From 9edcf177ce7210d7ee7f857dedf0747e5bdb4105 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 15 Sep 2011 22:20:45 -0700 Subject: [PATCH 001/349] storm starter --- .gitignore | 3 + README | 1 + multilang/resources/splitsentence.py | 9 ++ multilang/resources/storm.py | 143 ++++++++++++++++++ multilang/resources/storm.rb | 116 ++++++++++++++ project.clj | 19 +++ .../storm/starter/ExclamationTopology.java | 69 +++++++++ src/jvm/storm/starter/PrintSampleStream.java | 32 ++++ src/jvm/storm/starter/WordCountTopology.java | 84 ++++++++++ .../starter/spout/RandomSentenceSpout.java | 60 ++++++++ .../starter/spout/TwitterSampleSpout.java | 99 ++++++++++++ 11 files changed, 635 insertions(+) create mode 100644 .gitignore create mode 100644 README create mode 100644 multilang/resources/splitsentence.py create mode 100644 multilang/resources/storm.py create mode 100644 multilang/resources/storm.rb create mode 100644 project.clj create mode 100644 src/jvm/storm/starter/ExclamationTopology.java create mode 100644 src/jvm/storm/starter/PrintSampleStream.java create mode 100644 src/jvm/storm/starter/WordCountTopology.java create mode 100644 src/jvm/storm/starter/spout/RandomSentenceSpout.java create mode 100644 src/jvm/storm/starter/spout/TwitterSampleSpout.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..707a02d64e --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +classes/ +lib/ +*.pyc diff --git a/README b/README new file mode 100644 index 0000000000..ed74288d02 --- /dev/null +++ b/README @@ -0,0 +1 @@ +Example Storm topologies diff --git a/multilang/resources/splitsentence.py b/multilang/resources/splitsentence.py new file mode 100644 index 0000000000..24a9c9b82f --- /dev/null +++ b/multilang/resources/splitsentence.py @@ -0,0 +1,9 @@ +import storm + +class SplitSentenceBolt(storm.BasicBolt): + def process(self, tup): + words = tup.values[0].split(" ") + for word in words: + storm.emit([word]) + +SplitSentenceBolt().run() \ No newline at end of file diff --git a/multilang/resources/storm.py b/multilang/resources/storm.py new file mode 100644 index 0000000000..6bb6d04f12 --- /dev/null +++ b/multilang/resources/storm.py @@ -0,0 +1,143 @@ +import sys +import os +import traceback + +try: + import cjson + json_encode = cjson.encode + json_decode = lambda x: cjson.decode(x, all_unicode=True) +except ImportError: + import json + json_encode = lambda x: json.dumps(x, ensure_ascii=False) + json_decode = lambda x: json.loads(unicode(x)) + +def readStringMsg(): + msg = "" + while True: + line = sys.stdin.readline()[0:-1] + if line == "end": + break + msg = msg + line + "\n" + return msg[0:-1] + +ANCHOR_TUPLE = None + +#reads lines and reconstructs newlines appropriately +def readMsg(): + return json_decode(readStringMsg()) + +def sendToParent(s): + print s + print "end" + sys.stdout.flush() + +def sync(): + print "sync" + sys.stdout.flush() + +def sendpid(heartbeatdir): + pid = os.getpid() + print pid + sys.stdout.flush() + open(heartbeatdir + "/" + str(pid), "w").close() + +def sendMsgToParent(amap): + sendToParent(json_encode(amap)) + +def emittuple(tup, stream=None, anchors = [], directTask=None): + global ANCHOR_TUPLE + if ANCHOR_TUPLE is not None: + anchors = [ANCHOR_TUPLE] + m = {"command": "emit"} + if stream is not None: + m["stream"] = stream + m["anchors"] = map(lambda a: a.id, anchors) + if directTask is not None: + m["task"] = directTask + m["tuple"] = tup + sendMsgToParent(m) + +def emit(tup, stream=None, anchors = []): + emittuple(tup, stream=stream, anchors=anchors) + #read back task ids + return readMsg() + +def emitDirect(task, tup, stream=None, anchors = []): + emittuple(tup, stream=stream, anchors=anchors, directTask=task) + +def ack(tup): + sendMsgToParent({"command": "ack", "id": tup.id}) + +def fail(tup): + sendMsgToParent({"command": "fail", "id": tup.id}) + +def log(msg): + sendMsgToParent({"command": "log", "msg": msg}) + +# read the stormconf and context +def readenv(): + conf = readMsg() + context = readMsg() + return [conf, context] + +def readtuple(): + tupmap = readMsg() + return Tuple(tupmap["id"], tupmap["comp"], tupmap["stream"], tupmap["task"], tupmap["tuple"]) + +def initbolt(): + heartbeatdir = readStringMsg() + sendpid(heartbeatdir) + return readenv() + +class Tuple: + def __init__(self, id, component, stream, task, values): + self.id = id + self.component = component + self.stream = stream + self.task = task + self.values = values + +class Bolt: + def initialize(self, stormconf, context): + pass + + def process(self, tuple): + pass + + def run(self): + conf, context = initbolt() + self.initialize(conf, context) + try: + while True: + tup = readtuple() + self.process(tup) + sync() + except Exception, e: + log(traceback.format_exc(e)) + +class BasicBolt: + def initialize(self, stormconf, context): + pass + + def process(self, tuple): + pass + + def run(self): + global ANCHOR_TUPLE + conf, context = initbolt() + self.initialize(conf, context) + try: + while True: + tup = readtuple() + ANCHOR_TUPLE = tup + self.process(tup) + ack(tup) + sync() + except Exception, e: + log(traceback.format_exc(e)) + +class Spout: + pass + + + diff --git a/multilang/resources/storm.rb b/multilang/resources/storm.rb new file mode 100644 index 0000000000..32c17b631b --- /dev/null +++ b/multilang/resources/storm.rb @@ -0,0 +1,116 @@ +require "rubygems" +require "json" + +module Storm + module Protocol + def read_string_message + msg = "" + loop do + line = STDIN.readline.chomp + break if line == "end" + msg << line + msg << "\n" + end + msg.chomp + end + + def read_message + JSON.parse read_string_message + end + + def send_to_parent(s) + puts s + puts "end" + STDOUT.flush + end + + def sync + puts "sync" + STDOUT.flush + end + + def send_pid(heartbeat_dir) + pid = Process.pid + puts pid + STDOUT.flush + File.open("#{heartbeat_dir}/#{pid}", "w").close + end + + def send_msg_to_parent(hash) + send_to_parent hash.to_json + end + + def emit_tuple(tup, stream = nil, anchors = [], direct = nil) + m = {:command => :emit, :anchors => anchors.map(&:id), :tuple => tup} + m[:stream] = stream if stream + m[:task] = direct if direct + send_msg_to_parent m + end + + def emit(tup, stream = nil, anchors = [], direct = nil) + emit_tuple tup, stream, anchors, direct + read_message + end + + def ack(tup) + send_msg_to_parent :command => :ack, :id => tup.id + end + + def fail(tup) + send_msg_to_parent :command => :fail, :id => tup.id + end + + def log(msg) + send_msg_to_parent :command => :log, :msg => msg.to_s + end + + def read_env + [read_message, read_message] + end + end + + class Tuple + attr_accessor :id, :component, :stream, :task, :values + + def initialize(id, component, stream, task, values) + @id = id + @component = component + @stream = stream + @task = task + @values = values + end + + def self.from_hash(hash) + Tuple.new(*hash.values_at("id", "comp", "stream", "task", "tuple")) + end + end + + class Bolt + include Storm::Protocol + + def initialize(conf = nil, context = nil) + @conf = conf + @context = context + end + + def process(tuple) + end + + def run + heartbeat_dir = read_string_message + send_pid heartbeat_dir + @conf, @context = read_env + begin + while true + process Tuple.from_hash(read_message) + sync + end + rescue Exception => e + log e + end + end + end + + class Spout + end +end diff --git a/project.clj b/project.clj new file mode 100644 index 0000000000..2d536b7edb --- /dev/null +++ b/project.clj @@ -0,0 +1,19 @@ +(defproject storm-starter "0.0.1-SNAPSHOT" + :java-source-path "src/jvm" + :javac-options {:debug "true" :fork "true"} + :resources-path "multilang" + :aot :all + :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] + :repositories { + "twitter4j" "http://twitter4j.org/maven2" + } + + :dependencies [ + [backtype/twitter4j-core "2.2.4-SNAPSHOT"] + [org.twitter4j/twitter4j-stream "2.2.4-SNAPSHOT"] + [com.googlecode.json-simple/json-simple "1.1"] + ] + + :dev-dependencies [[storm "0.5.0"] + ]) + diff --git a/src/jvm/storm/starter/ExclamationTopology.java b/src/jvm/storm/starter/ExclamationTopology.java new file mode 100644 index 0000000000..91bf30e0a4 --- /dev/null +++ b/src/jvm/storm/starter/ExclamationTopology.java @@ -0,0 +1,69 @@ +package storm.starter; + +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.task.OutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.testing.TestWordSpout; +import backtype.storm.topology.IRichBolt; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.TopologyBuilder; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; +import backtype.storm.utils.Utils; +import java.util.HashMap; +import java.util.Map; + + +public class ExclamationTopology { + + public static class ExclamationBolt implements IRichBolt { + OutputCollector _collector; + + @Override + public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + _collector = collector; + } + + @Override + public void execute(Tuple tuple) { + _collector.emit(tuple, new Values(tuple.getString(0) + "!!!")); + _collector.ack(tuple); + } + + @Override + public void cleanup() { + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("word")); + } + + + } + + public static void main(String[] args) { + TopologyBuilder builder = new TopologyBuilder(); + + builder.setSpout(1, new TestWordSpout(), 10); + + builder.setBolt(2, new ExclamationBolt(), 3) + .shuffleGrouping(1); + + + + + + Map conf = new HashMap(); + conf.put(Config.TOPOLOGY_DEBUG, true); + + + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology("test", conf, builder.createTopology()); + Utils.sleep(10000); + cluster.killTopology("test"); + cluster.shutdown(); + } +} diff --git a/src/jvm/storm/starter/PrintSampleStream.java b/src/jvm/storm/starter/PrintSampleStream.java new file mode 100644 index 0000000000..c196b8d862 --- /dev/null +++ b/src/jvm/storm/starter/PrintSampleStream.java @@ -0,0 +1,32 @@ +package storm.starter; + +import storm.starter.spout.TwitterSampleSpout; +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.topology.TopologyBuilder; +import backtype.storm.utils.Utils; +import java.util.HashMap; +import java.util.Map; + + +public class PrintSampleStream { + public static void main(String[] args) { + String username = args[0]; + String pwd = args[1]; + TopologyBuilder builder = new TopologyBuilder(); + + builder.setSpout(1, new TwitterSampleSpout(username, pwd)); + + + Map conf = new HashMap(); + conf.put(Config.TOPOLOGY_DEBUG, true); + + + LocalCluster cluster = new LocalCluster(); + + cluster.submitTopology("test", conf, builder.createTopology()); + + Utils.sleep(10000); + cluster.shutdown(); + } +} diff --git a/src/jvm/storm/starter/WordCountTopology.java b/src/jvm/storm/starter/WordCountTopology.java new file mode 100644 index 0000000000..74a0caddb5 --- /dev/null +++ b/src/jvm/storm/starter/WordCountTopology.java @@ -0,0 +1,84 @@ +package storm.starter; + +import storm.starter.spout.RandomSentenceSpout; +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.task.ShellBolt; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.BasicOutputCollector; +import backtype.storm.topology.IBasicBolt; +import backtype.storm.topology.IRichBolt; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.TopologyBuilder; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; +import java.util.HashMap; +import java.util.Map; + + +public class WordCountTopology { + public static class SplitSentence extends ShellBolt implements IRichBolt { + + public SplitSentence() { + super("python", "splitsentence.py"); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("word")); + } + } + + public static class WordCount implements IBasicBolt { + Map counts = new HashMap(); + + @Override + public void prepare(Map conf, TopologyContext context) { + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + String word = tuple.getString(0); + Integer count = counts.get(word); + if(count==null) count = 0; + count++; + counts.put(word, count); + collector.emit(new Values(word, count)); + } + + @Override + public void cleanup() { + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("word", "count")); + } + } + + public static void main(String[] args) throws Exception { + + TopologyBuilder builder = new TopologyBuilder(); + + builder.setSpout(1, new RandomSentenceSpout(), 5); + + builder.setBolt(2, new SplitSentence(), 8) + .shuffleGrouping(1); + builder.setBolt(3, new WordCount(), 12) + .fieldsGrouping(2, new Fields("word")); + + + + Map conf = new HashMap(); + conf.put(Config.TOPOLOGY_DEBUG, true); + conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 3); + + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology("word-count", conf, builder.createTopology()); + Thread.sleep(10000); + + cluster.shutdown(); + + } +} diff --git a/src/jvm/storm/starter/spout/RandomSentenceSpout.java b/src/jvm/storm/starter/spout/RandomSentenceSpout.java new file mode 100644 index 0000000000..1a48db1a99 --- /dev/null +++ b/src/jvm/storm/starter/spout/RandomSentenceSpout.java @@ -0,0 +1,60 @@ +package storm.starter.spout; + +import backtype.storm.spout.SpoutOutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.IRichSpout; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Values; +import backtype.storm.utils.Utils; +import java.util.Map; +import java.util.Random; + +public class RandomSentenceSpout implements IRichSpout { + SpoutOutputCollector _collector; + Random _rand; + + @Override + public boolean isDistributed() { + return true; + } + + @Override + public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + _collector = collector; + _rand = new Random(); + } + + @Override + public void nextTuple() { + Utils.sleep(100); + String[] sentences = new String[] { + "the cow jumped over the moon", + "an apple a day keeps the doctor away", + "four score and seven years ago", + "snow white and the seven dwarfs", + "i am at two with nature"}; + String sentence = sentences[_rand.nextInt(sentences.length)]; + _collector.emit(new Values(sentence)); + } + + + @Override + public void close() { + } + + + @Override + public void ack(Object id) { + } + + @Override + public void fail(Object id) { + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("word")); + } + +} \ No newline at end of file diff --git a/src/jvm/storm/starter/spout/TwitterSampleSpout.java b/src/jvm/storm/starter/spout/TwitterSampleSpout.java new file mode 100644 index 0000000000..a443862c7e --- /dev/null +++ b/src/jvm/storm/starter/spout/TwitterSampleSpout.java @@ -0,0 +1,99 @@ +package storm.starter.spout; + +import twitter4j.conf.ConfigurationBuilder; +import twitter4j.TwitterStream; +import twitter4j.TwitterStreamFactory; +import backtype.storm.spout.SpoutOutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.IRichSpout; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.tuple.Fields; +import backtype.storm.utils.Utils; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import twitter4j.Status; +import twitter4j.StatusDeletionNotice; +import twitter4j.StatusListener; +import static backtype.storm.utils.Utils.*; + +public class TwitterSampleSpout implements IRichSpout { + SpoutOutputCollector _collector; + LinkedBlockingQueue queue = null; + TwitterStream _twitterStream; + String _username; + String _pwd; + + + public TwitterSampleSpout(String username, String pwd) { + _username = username; + _pwd = pwd; + } + + @Override + public boolean isDistributed() { + return false; + } + + @Override + public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + queue = new LinkedBlockingQueue(1000); + _collector = collector; + StatusListener listener = new StatusListener() { + + @Override + public void onStatus(Status status) { + queue.offer(status.toString()); + } + + @Override + public void onDeletionNotice(StatusDeletionNotice sdn) { + } + + @Override + public void onTrackLimitationNotice(int i) { + } + + @Override + public void onScrubGeo(long l, long l1) { + } + + @Override + public void onException(Exception e) { + } + + }; + TwitterStreamFactory fact = new TwitterStreamFactory(new ConfigurationBuilder().setUser(_username).setPassword(_pwd).build()); + _twitterStream = fact.getInstance(); + _twitterStream.addListener(listener); + _twitterStream.sample(); + } + + @Override + public void nextTuple() { + String ret = queue.poll(); + if(ret==null) { + Utils.sleep(50); + } else { + _collector.emit(tuple(ret)); + } + } + + @Override + public void close() { + _twitterStream.shutdown(); + } + + @Override + public void ack(Object id) { + } + + @Override + public void fail(Object id) { + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("tweet")); + } + +} From c80b08866d7730d78fb7c0edfc8dc2d758942921 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 16 Sep 2011 12:06:14 -0700 Subject: [PATCH 002/349] add license --- LICENSE | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..19560d1c38 --- /dev/null +++ b/LICENSE @@ -0,0 +1,14 @@ +Copyright 2011 Nathan Marz + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + From e1a6615407915402f43554429b88ab75670bc99b Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 21 Sep 2011 11:51:04 -0500 Subject: [PATCH 003/349] add clojure dependencies so people can try out storm-starter from repl --- project.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 2d536b7edb..e99fd96fe6 100644 --- a/project.clj +++ b/project.clj @@ -8,7 +8,8 @@ "twitter4j" "http://twitter4j.org/maven2" } - :dependencies [ + :dependencies [[org.clojure/clojure "1.2.0"] + [org.clojure/clojure-contrib "1.2.0"] [backtype/twitter4j-core "2.2.4-SNAPSHOT"] [org.twitter4j/twitter4j-stream "2.2.4-SNAPSHOT"] [com.googlecode.json-simple/json-simple "1.1"] From 433caaafb04835095cfbaed5e565dfa65d32bb3c Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 21 Sep 2011 20:15:48 -0700 Subject: [PATCH 004/349] upgrade to 0.5.1 API --- .gitignore | 2 ++ project.clj | 2 +- src/jvm/storm/starter/ExclamationTopology.java | 14 +++++--------- src/jvm/storm/starter/PrintSampleStream.java | 6 ++---- src/jvm/storm/starter/WordCountTopology.java | 6 +++--- 5 files changed, 13 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 707a02d64e..5f30b63ed0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ classes/ lib/ *.pyc +.classpath +.project diff --git a/project.clj b/project.clj index e99fd96fe6..6f2abd0795 100644 --- a/project.clj +++ b/project.clj @@ -15,6 +15,6 @@ [com.googlecode.json-simple/json-simple "1.1"] ] - :dev-dependencies [[storm "0.5.0"] + :dev-dependencies [[storm "0.5.1"] ]) diff --git a/src/jvm/storm/starter/ExclamationTopology.java b/src/jvm/storm/starter/ExclamationTopology.java index 91bf30e0a4..3eeb6a14ea 100644 --- a/src/jvm/storm/starter/ExclamationTopology.java +++ b/src/jvm/storm/starter/ExclamationTopology.java @@ -12,7 +12,6 @@ import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; import backtype.storm.utils.Utils; -import java.util.HashMap; import java.util.Map; @@ -47,17 +46,14 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { public static void main(String[] args) { TopologyBuilder builder = new TopologyBuilder(); - builder.setSpout(1, new TestWordSpout(), 10); - + builder.setSpout(1, new TestWordSpout(), 10); builder.setBolt(2, new ExclamationBolt(), 3) .shuffleGrouping(1); - - + builder.setBolt(3, new ExclamationBolt(), 2) + .shuffleGrouping(2); - - - Map conf = new HashMap(); - conf.put(Config.TOPOLOGY_DEBUG, true); + Config conf = new Config(); + conf.setDebug(true); LocalCluster cluster = new LocalCluster(); diff --git a/src/jvm/storm/starter/PrintSampleStream.java b/src/jvm/storm/starter/PrintSampleStream.java index c196b8d862..8a8445a255 100644 --- a/src/jvm/storm/starter/PrintSampleStream.java +++ b/src/jvm/storm/starter/PrintSampleStream.java @@ -5,8 +5,6 @@ import backtype.storm.LocalCluster; import backtype.storm.topology.TopologyBuilder; import backtype.storm.utils.Utils; -import java.util.HashMap; -import java.util.Map; public class PrintSampleStream { @@ -18,8 +16,8 @@ public static void main(String[] args) { builder.setSpout(1, new TwitterSampleSpout(username, pwd)); - Map conf = new HashMap(); - conf.put(Config.TOPOLOGY_DEBUG, true); + Config conf = new Config(); + conf.setDebug(true); LocalCluster cluster = new LocalCluster(); diff --git a/src/jvm/storm/starter/WordCountTopology.java b/src/jvm/storm/starter/WordCountTopology.java index 74a0caddb5..f578d2eb10 100644 --- a/src/jvm/storm/starter/WordCountTopology.java +++ b/src/jvm/storm/starter/WordCountTopology.java @@ -70,9 +70,9 @@ public static void main(String[] args) throws Exception { - Map conf = new HashMap(); - conf.put(Config.TOPOLOGY_DEBUG, true); - conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 3); + Config conf = new Config(); + conf.setDebug(true); + conf.setMaxTaskParallelism(3); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("word-count", conf, builder.createTopology()); From 7027ea9cf508c32be56e064182d8d1035d27bfe3 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 24 Sep 2011 12:13:27 -0700 Subject: [PATCH 005/349] bump storm dependency for no zmq local mode --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 6f2abd0795..bbc13e349f 100644 --- a/project.clj +++ b/project.clj @@ -15,6 +15,6 @@ [com.googlecode.json-simple/json-simple "1.1"] ] - :dev-dependencies [[storm "0.5.1"] + :dev-dependencies [[storm "0.5.2"] ]) From 7fa873d10e147e2363c064f0c3ca64893022e51a Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 28 Sep 2011 00:35:52 -0700 Subject: [PATCH 006/349] added example of clojure DSL --- project.clj | 3 +- src/clj/storm/starter/clj/word_count.clj | 60 ++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 src/clj/storm/starter/clj/word_count.clj diff --git a/project.clj b/project.clj index bbc13e349f..1ba16324bc 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,5 @@ (defproject storm-starter "0.0.1-SNAPSHOT" + :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} :resources-path "multilang" @@ -15,6 +16,6 @@ [com.googlecode.json-simple/json-simple "1.1"] ] - :dev-dependencies [[storm "0.5.2"] + :dev-dependencies [[storm "0.5.3-SNAPSHOT"] ]) diff --git a/src/clj/storm/starter/clj/word_count.clj b/src/clj/storm/starter/clj/word_count.clj new file mode 100644 index 0000000000..41f87e19d6 --- /dev/null +++ b/src/clj/storm/starter/clj/word_count.clj @@ -0,0 +1,60 @@ +(ns storm.starter.clj.word-count + (:import backtype.storm.LocalCluster) + (:use [backtype.storm clojure config])) + +(defspout sentence-spout ["sentence"] + [conf context collector] + (let [sentences ["a little brown dog" + "the man petted the dog" + "four score and seven years ago" + "an apple a day keeps the doctor away"]] + (spout + (nextTuple [] + (Thread/sleep 100) + (emit-spout! collector [(rand-nth sentences)]) + ) + (ack [id] + ;; You only need to define this method for reliable spouts + ;; (such as one that reads off of a queue like Kestrel) + ;; This is an unreliable spout, so it does nothing here + )))) + +(defspout sentence-spout-parameterized ["word"] {:params [sentences] :prepare false} + [collector] + (Thread/sleep 500) + (emit-spout! collector [(rand-nth sentences)])) + +(defbolt split-sentence ["word"] [tuple collector] + (let [words (.split (.getString tuple 0) " ")] + (doseq [w words] + (emit-bolt! collector [w] :anchor tuple)) + (ack! collector tuple) + )) + +(defbolt word-count ["word" "count"] {:prepare true} + [conf context collector] + (let [counts (atom {})] + (bolt + (execute [tuple] + (let [word (.getString tuple 0)] + (swap! counts (partial merge-with +) {word 1}) + (emit-bolt! collector [word (@counts word)] :anchor tuple) + (ack! collector tuple) + ))))) + +(defn mk-topology [] + (topology + {1 (spout-spec sentence-spout) + 2 (spout-spec (sentence-spout-parameterized + ["the cat jumped over the door" + "greetings from a faraway land"]) + :p 2)} + {3 (bolt-spec {1 :shuffle 2 :shuffle} split-sentence :p 5) + 4 (bolt-spec {3 ["word"]} word-count :p 6)})) + +(defn run-local! [] + (let [cluster (LocalCluster.)] + (.submitTopology cluster "word-count" {TOPOLOGY-DEBUG true} (mk-topology)) + (Thread/sleep 10000) + (.shutdown cluster) + )) From 3c4f437f59c3a52b2e48a4e38b2072f05fb58262 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 28 Sep 2011 16:30:20 -0700 Subject: [PATCH 007/349] update storm dependency --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 1ba16324bc..a347bb771a 100644 --- a/project.clj +++ b/project.clj @@ -16,6 +16,6 @@ [com.googlecode.json-simple/json-simple "1.1"] ] - :dev-dependencies [[storm "0.5.3-SNAPSHOT"] + :dev-dependencies [[storm "0.5.3"] ]) From 4aa082dbefd01d72aa745a9b6593bad2bc6352d6 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 29 Sep 2011 18:42:35 -0700 Subject: [PATCH 008/349] formatting --- src/clj/storm/starter/clj/word_count.clj | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/clj/storm/starter/clj/word_count.clj b/src/clj/storm/starter/clj/word_count.clj index 41f87e19d6..5c0a47291d 100644 --- a/src/clj/storm/starter/clj/word_count.clj +++ b/src/clj/storm/starter/clj/word_count.clj @@ -1,6 +1,7 @@ (ns storm.starter.clj.word-count - (:import backtype.storm.LocalCluster) - (:use [backtype.storm clojure config])) + (:import [backtype.storm StormSubmitter LocalCluster]) + (:use [backtype.storm clojure config]) + (:gen-class)) (defspout sentence-spout ["sentence"] [conf context collector] @@ -43,14 +44,19 @@ ))))) (defn mk-topology [] + (topology {1 (spout-spec sentence-spout) 2 (spout-spec (sentence-spout-parameterized ["the cat jumped over the door" "greetings from a faraway land"]) :p 2)} - {3 (bolt-spec {1 :shuffle 2 :shuffle} split-sentence :p 5) - 4 (bolt-spec {3 ["word"]} word-count :p 6)})) + {3 (bolt-spec {1 :shuffle 2 :shuffle} + split-sentence + :p 5) + 4 (bolt-spec {3 ["word"]} + word-count + :p 6)})) (defn run-local! [] (let [cluster (LocalCluster.)] @@ -58,3 +64,11 @@ (Thread/sleep 10000) (.shutdown cluster) )) + +(defn -main [name] + (StormSubmitter/submitTopology + name + {TOPOLOGY-DEBUG true + TOPOLOGY-WORKERS 3} + (mk-topology))) + From ef3020a425fcb3e4fb85dcf107b9d4c0ba7a14b2 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 29 Sep 2011 18:43:02 -0700 Subject: [PATCH 009/349] example of doing scalable streaming rolling top N --- src/jvm/storm/starter/RollingTopWords.java | 41 +++++++ src/jvm/storm/starter/bolt/MergeObjects.java | 110 +++++++++++++++++ src/jvm/storm/starter/bolt/RankObjects.java | 105 ++++++++++++++++ .../starter/bolt/RollingCountObjects.java | 113 ++++++++++++++++++ 4 files changed, 369 insertions(+) create mode 100644 src/jvm/storm/starter/RollingTopWords.java create mode 100644 src/jvm/storm/starter/bolt/MergeObjects.java create mode 100644 src/jvm/storm/starter/bolt/RankObjects.java create mode 100644 src/jvm/storm/starter/bolt/RollingCountObjects.java diff --git a/src/jvm/storm/starter/RollingTopWords.java b/src/jvm/storm/starter/RollingTopWords.java new file mode 100644 index 0000000000..c39a918ad7 --- /dev/null +++ b/src/jvm/storm/starter/RollingTopWords.java @@ -0,0 +1,41 @@ +package storm.starter; + +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.testing.TestWordSpout; +import backtype.storm.topology.TopologyBuilder; +import backtype.storm.tuple.Fields; +import storm.starter.bolt.MergeObjects; +import storm.starter.bolt.RankObjects; +import storm.starter.bolt.RollingCountObjects; + +public class RollingTopWords { + + public static void main(String[] args) throws Exception { + + final int TOP_N = 3; + + TopologyBuilder builder = new TopologyBuilder(); + + builder.setSpout(1, new TestWordSpout(), 5); + + builder.setBolt(2, new RollingCountObjects(60, 10), 4) + .fieldsGrouping(1, new Fields("word")); + builder.setBolt(3, new RankObjects(TOP_N), 4) + .fieldsGrouping(2, new Fields("obj")); + builder.setBolt(4, new MergeObjects(TOP_N)) + .globalGrouping(3); + + + + Config conf = new Config(); + conf.setDebug(true); + + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology("rolling-demo", conf, builder.createTopology()); + Thread.sleep(10000); + + cluster.shutdown(); + + } +} diff --git a/src/jvm/storm/starter/bolt/MergeObjects.java b/src/jvm/storm/starter/bolt/MergeObjects.java new file mode 100644 index 0000000000..aa54d6cb13 --- /dev/null +++ b/src/jvm/storm/starter/bolt/MergeObjects.java @@ -0,0 +1,110 @@ +package storm.starter.bolt; + +import org.apache.log4j.Logger; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.BasicOutputCollector; +import backtype.storm.topology.IBasicBolt; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import org.json.simple.JSONValue; + + +public class MergeObjects implements IBasicBolt { + public static Logger LOG = Logger.getLogger(MergeObjects.class); + + private List _rankings = new ArrayList(); + int _count = 10; + Long _lastTime; + + public MergeObjects(int n) { + _count = n; + } + + + private int _compare(List one, List two) { + + long valueOne = (Long) one.get(1); + long valueTwo = (Long) two.get(1); + + long delta = valueTwo - valueOne; + if(delta > 0) { + return 1; + } else if (delta < 0) { + return -1; + } else { + return 0; + } + + } //end compare + + private Integer _find(Object tag) { + for(int i = 0; i < _rankings.size(); ++i) { + + Object cur = _rankings.get(i).get(0); + if (cur.equals(tag)) { + return i; + } + + } + + return null; + + } + + + public void prepare(Map stormConf, TopologyContext context) { + + } + + public void execute(Tuple tuple, BasicOutputCollector collector) { + + + List merging = (List) JSONValue.parse(tuple.getString(0)); + for(List pair : merging) { + + Integer existingIndex = _find(pair.get(0)); + if (null != existingIndex) { + _rankings.set(existingIndex, pair); + } else { + + _rankings.add(pair); + + } + + Collections.sort(_rankings, new Comparator() { + public int compare(List o1, List o2) { + return _compare(o1, o2); + } + }); + + + if (_rankings.size() > _count) { + _rankings.subList(_count, _rankings.size()).clear(); + } + + } + + long currentTime = System.currentTimeMillis(); + if(_lastTime==null || currentTime >= _lastTime + 2000) { + String fullRankings = JSONValue.toJSONString(_rankings); + collector.emit(new Values(fullRankings)); + LOG.info("Rankings: " + fullRankings); + _lastTime = currentTime; + } + } + + public void cleanup() { + } + + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("list")); + } + +} diff --git a/src/jvm/storm/starter/bolt/RankObjects.java b/src/jvm/storm/starter/bolt/RankObjects.java new file mode 100644 index 0000000000..68ab325fc4 --- /dev/null +++ b/src/jvm/storm/starter/bolt/RankObjects.java @@ -0,0 +1,105 @@ +package storm.starter.bolt; + +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.BasicOutputCollector; +import backtype.storm.topology.IBasicBolt; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import org.json.simple.JSONValue; + + +public class RankObjects implements IBasicBolt { + + List _rankings = new ArrayList(); + + int _count; + Long _lastTime = null; + + public RankObjects(int n) { + _count = n; + } + + + private int _compare(List one, List two) { + + long valueOne = (Long) one.get(1); + long valueTwo = (Long) two.get(1); + + long delta = valueTwo - valueOne; + if(delta > 0) { + return 1; + } else if (delta < 0) { + return -1; + } else { + return 0; + } + + } //end compare + + private Integer _find(Object tag) { + for(int i = 0; i < _rankings.size(); ++i) { + + Object cur = _rankings.get(i).get(0); + if (cur.equals(tag)) { + return i; + } + + } + + return null; + + } + + public void prepare(Map stormConf, TopologyContext context) { + + } + + public void execute(Tuple tuple, BasicOutputCollector collector) { + + Object tag = tuple.getValue(0); + + + Integer existingIndex = _find(tag); + if (null != existingIndex) { + _rankings.set(existingIndex, tuple.getValues()); + } else { + + _rankings.add(tuple.getValues()); + + + } + + + Collections.sort(_rankings, new Comparator() { + public int compare(List o1, List o2) { + return _compare(o1, o2); + } + }); + + + if (_rankings.size() > _count) { + _rankings.remove(_count); + } + + long currentTime = System.currentTimeMillis(); + if(_lastTime==null || currentTime >= _lastTime + 2000) { + collector.emit(new Values(JSONValue.toJSONString(_rankings))); + _lastTime = currentTime; + } + } + + public void cleanup() { + } + + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("list")); + } + +} diff --git a/src/jvm/storm/starter/bolt/RollingCountObjects.java b/src/jvm/storm/starter/bolt/RollingCountObjects.java new file mode 100644 index 0000000000..55d73e013a --- /dev/null +++ b/src/jvm/storm/starter/bolt/RollingCountObjects.java @@ -0,0 +1,113 @@ +package storm.starter.bolt; + +import backtype.storm.task.OutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.IRichBolt; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; +import backtype.storm.utils.Utils; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class RollingCountObjects implements IRichBolt { + + private HashMap _objectCounts = new HashMap(); + private int _numBuckets; + private transient Thread cleaner; + private OutputCollector _collector; + private int _trackMinutes; + + public RollingCountObjects(int numBuckets, int trackMinutes) { + _numBuckets = numBuckets; + _trackMinutes = trackMinutes; + } + + public long totalObjects (Object obj) { + long[] curr = _objectCounts.get(obj); + long total = 0; + for (long l: curr) { + total+=l; + } + return total; + } + + public int currentBucket (int buckets) { + return (currentSecond() / secondsPerBucket(buckets)) % buckets; + } + + public int currentSecond() { + return (int) (System.currentTimeMillis() / 1000); + } + + public int secondsPerBucket(int buckets) { + return (_trackMinutes * 60 / buckets); + } + + public long millisPerBucket(int buckets) { + return (long) secondsPerBucket(buckets) * 1000; + } + + public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { + _collector = collector; + cleaner = new Thread(new Runnable() { + public void run() { + Integer lastBucket = currentBucket(_numBuckets); + + + while(true) { + int currBucket = currentBucket(_numBuckets); + if(currBucket!=lastBucket) { + int bucketToWipe = (currBucket + 1) % _numBuckets; + synchronized(_objectCounts) { + Set objs = new HashSet(_objectCounts.keySet()); + for (Object obj: objs) { + long[] counts = _objectCounts.get(obj); + long currBucketVal = counts[bucketToWipe]; + counts[bucketToWipe] = 0; + long total = totalObjects(obj); + if(currBucketVal!=0) { + _collector.emit(new Values(obj, total)); + } + if(total==0) { + _objectCounts.remove(obj); + } + } + } + lastBucket = currBucket; + } + long delta = millisPerBucket(_numBuckets) - (System.currentTimeMillis() % millisPerBucket(_numBuckets)); + Utils.sleep(delta); + } + } + }); + cleaner.start(); + } + + public void execute(Tuple tuple) { + + Object obj = tuple.getValue(0); + int bucket = currentBucket(_numBuckets); + synchronized(_objectCounts) { + long[] curr = _objectCounts.get(obj); + if(curr==null) { + curr = new long[_numBuckets]; + _objectCounts.put(obj, curr); + } + curr[bucket]++; + _collector.emit(new Values(obj, totalObjects(obj))); + _collector.ack(tuple); + } + } + + public void cleanup() { + } + + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("obj", "count")); + } + +} From 9d267e8da0b9ab81aecc39903517beeaaaf2ea53 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 30 Sep 2011 13:50:45 -0700 Subject: [PATCH 010/349] tuple() -> new Values() --- src/jvm/storm/starter/spout/TwitterSampleSpout.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jvm/storm/starter/spout/TwitterSampleSpout.java b/src/jvm/storm/starter/spout/TwitterSampleSpout.java index a443862c7e..28e2551928 100644 --- a/src/jvm/storm/starter/spout/TwitterSampleSpout.java +++ b/src/jvm/storm/starter/spout/TwitterSampleSpout.java @@ -8,13 +8,13 @@ import backtype.storm.topology.IRichSpout; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Values; import backtype.storm.utils.Utils; import java.util.Map; import java.util.concurrent.LinkedBlockingQueue; import twitter4j.Status; import twitter4j.StatusDeletionNotice; import twitter4j.StatusListener; -import static backtype.storm.utils.Utils.*; public class TwitterSampleSpout implements IRichSpout { SpoutOutputCollector _collector; @@ -74,7 +74,7 @@ public void nextTuple() { if(ret==null) { Utils.sleep(50); } else { - _collector.emit(tuple(ret)); + _collector.emit(new Values(ret)); } } From d353e6be58ac830e7cd20a10c05d84d2651cce2a Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Tue, 4 Oct 2011 03:38:02 -0700 Subject: [PATCH 011/349] fix TwitterSampleSpout --- project.clj | 2 +- src/jvm/storm/starter/spout/TwitterSampleSpout.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index a347bb771a..62959a3314 100644 --- a/project.clj +++ b/project.clj @@ -11,7 +11,7 @@ :dependencies [[org.clojure/clojure "1.2.0"] [org.clojure/clojure-contrib "1.2.0"] - [backtype/twitter4j-core "2.2.4-SNAPSHOT"] + [org.twitter4j/twitter4j-core "2.2.4-SNAPSHOT"] [org.twitter4j/twitter4j-stream "2.2.4-SNAPSHOT"] [com.googlecode.json-simple/json-simple "1.1"] ] diff --git a/src/jvm/storm/starter/spout/TwitterSampleSpout.java b/src/jvm/storm/starter/spout/TwitterSampleSpout.java index 28e2551928..993e01de43 100644 --- a/src/jvm/storm/starter/spout/TwitterSampleSpout.java +++ b/src/jvm/storm/starter/spout/TwitterSampleSpout.java @@ -15,6 +15,7 @@ import twitter4j.Status; import twitter4j.StatusDeletionNotice; import twitter4j.StatusListener; +import twitter4j.json.DataObjectFactory; public class TwitterSampleSpout implements IRichSpout { SpoutOutputCollector _collector; @@ -41,8 +42,8 @@ public void open(Map conf, TopologyContext context, SpoutOutputCollector collect StatusListener listener = new StatusListener() { @Override - public void onStatus(Status status) { - queue.offer(status.toString()); + public void onStatus(Status status) { + queue.offer(DataObjectFactory.getRawJSON(status)); } @Override From a17ae044738eb82ec8a93331156fe539dc6ad929 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 20 Oct 2011 01:19:16 -0700 Subject: [PATCH 012/349] make wordcounttopology and exclamationtopology run in remote mode --- .../storm/starter/ExclamationTopology.java | 19 +++++++++----- src/jvm/storm/starter/WordCountTopology.java | 26 ++++++++++++------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/jvm/storm/starter/ExclamationTopology.java b/src/jvm/storm/starter/ExclamationTopology.java index 3eeb6a14ea..8a06d96655 100644 --- a/src/jvm/storm/starter/ExclamationTopology.java +++ b/src/jvm/storm/starter/ExclamationTopology.java @@ -2,6 +2,7 @@ import backtype.storm.Config; import backtype.storm.LocalCluster; +import backtype.storm.StormSubmitter; import backtype.storm.task.OutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.testing.TestWordSpout; @@ -43,7 +44,7 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } - public static void main(String[] args) { + public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(1, new TestWordSpout(), 10); @@ -55,11 +56,17 @@ public static void main(String[] args) { Config conf = new Config(); conf.setDebug(true); + if(args.length > 0) { + conf.setNumWorkers(3); + + StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); + } else { - LocalCluster cluster = new LocalCluster(); - cluster.submitTopology("test", conf, builder.createTopology()); - Utils.sleep(10000); - cluster.killTopology("test"); - cluster.shutdown(); + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology("test", conf, builder.createTopology()); + Utils.sleep(10000); + cluster.killTopology("test"); + cluster.shutdown(); + } } } diff --git a/src/jvm/storm/starter/WordCountTopology.java b/src/jvm/storm/starter/WordCountTopology.java index f578d2eb10..400275c6d5 100644 --- a/src/jvm/storm/starter/WordCountTopology.java +++ b/src/jvm/storm/starter/WordCountTopology.java @@ -3,6 +3,7 @@ import storm.starter.spout.RandomSentenceSpout; import backtype.storm.Config; import backtype.storm.LocalCluster; +import backtype.storm.StormSubmitter; import backtype.storm.task.ShellBolt; import backtype.storm.task.TopologyContext; import backtype.storm.topology.BasicOutputCollector; @@ -67,18 +68,23 @@ public static void main(String[] args) throws Exception { .shuffleGrouping(1); builder.setBolt(3, new WordCount(), 12) .fieldsGrouping(2, new Fields("word")); - - - + Config conf = new Config(); conf.setDebug(true); - conf.setMaxTaskParallelism(3); - - LocalCluster cluster = new LocalCluster(); - cluster.submitTopology("word-count", conf, builder.createTopology()); - Thread.sleep(10000); - - cluster.shutdown(); + + if(args.length > 0) { + conf.setNumWorkers(3); + + StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); + } else { + conf.setMaxTaskParallelism(3); + + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology("word-count", conf, builder.createTopology()); + Thread.sleep(10000); + + cluster.shutdown(); + } } } From a1ac4b45a25de4cb3832940df36b268ce5f2ccb1 Mon Sep 17 00:00:00 2001 From: Bertrand Dechoux Date: Thu, 20 Oct 2011 12:40:10 +0300 Subject: [PATCH 013/349] 2.2.5 is required for Twitter (Kurtis remark) see https://github.com/BertrandDechoux/storm-starter/blob/java/pom.xml see http://groups.google.com/group/storm-user/browse_thread/thread/0faa0f7f42d7afe3 --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 62959a3314..b46b8c09e9 100644 --- a/project.clj +++ b/project.clj @@ -11,8 +11,8 @@ :dependencies [[org.clojure/clojure "1.2.0"] [org.clojure/clojure-contrib "1.2.0"] - [org.twitter4j/twitter4j-core "2.2.4-SNAPSHOT"] - [org.twitter4j/twitter4j-stream "2.2.4-SNAPSHOT"] + [org.twitter4j/twitter4j-core "2.2.5-SNAPSHOT"] + [org.twitter4j/twitter4j-stream "2.2.5-SNAPSHOT"] [com.googlecode.json-simple/json-simple "1.1"] ] From c2919ad2e485f85f85b57eeebb20b50593a5175f Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sun, 23 Oct 2011 15:29:55 -0700 Subject: [PATCH 014/349] added drpc examples --- project.clj | 2 +- src/jvm/storm/starter/BasicDRPCTopology.java | 68 ++++++ .../storm/starter/ExclamationTopology.java | 4 +- src/jvm/storm/starter/ReachTopology.java | 226 ++++++++++++++++++ src/jvm/storm/starter/RollingTopWords.java | 7 + src/jvm/storm/starter/WordCountTopology.java | 4 +- 6 files changed, 308 insertions(+), 3 deletions(-) create mode 100644 src/jvm/storm/starter/BasicDRPCTopology.java create mode 100644 src/jvm/storm/starter/ReachTopology.java diff --git a/project.clj b/project.clj index 62959a3314..d3477f2fa1 100644 --- a/project.clj +++ b/project.clj @@ -16,6 +16,6 @@ [com.googlecode.json-simple/json-simple "1.1"] ] - :dev-dependencies [[storm "0.5.3"] + :dev-dependencies [[storm "0.5.4"] ]) diff --git a/src/jvm/storm/starter/BasicDRPCTopology.java b/src/jvm/storm/starter/BasicDRPCTopology.java new file mode 100644 index 0000000000..fcd6ee575d --- /dev/null +++ b/src/jvm/storm/starter/BasicDRPCTopology.java @@ -0,0 +1,68 @@ +package storm.starter; + +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.LocalDRPC; +import backtype.storm.StormSubmitter; +import backtype.storm.drpc.LinearDRPCTopologyBuilder; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.BasicOutputCollector; +import backtype.storm.topology.IBasicBolt; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; +import java.util.Map; + +/** + * This topology is a basic example of doing distributed RPC on top of Storm. It implements a function + * that appends a "!" to any string you send the DRPC function. + * + * See https://github.com/nathanmarz/storm/wiki/Distributed-RPC for more information on + * doing distributed RPC on top of Storm. + */ +public class BasicDRPCTopology { + public static class ExclaimBolt implements IBasicBolt { + public void prepare(Map conf, TopologyContext context) { + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + String input = tuple.getString(1); + collector.emit(new Values(tuple.getValue(0), input + "!")); + } + + @Override + public void cleanup() { + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "result")); + } + + } + + public static void main(String[] args) throws Exception { + LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("exclamation"); + builder.addBolt(new ExclaimBolt(), 3); + + Config conf = new Config(); + conf.setDebug(true); + + if(args.length==0) { + LocalDRPC drpc = new LocalDRPC(); + LocalCluster cluster = new LocalCluster(); + + cluster.submitTopology("drpc-demo", conf, builder.createLocalTopology(drpc)); + + for(String word: new String[] {"hello", "goodbye"}) { + System.out.println("Result for \"" + word + "\": " + + drpc.execute("exclamation", word)); + } + } else { + conf.setNumWorkers(3); + StormSubmitter.submitTopology(args[0], conf, builder.createRemoteTopology()); + } + } +} diff --git a/src/jvm/storm/starter/ExclamationTopology.java b/src/jvm/storm/starter/ExclamationTopology.java index 8a06d96655..c6080f63a3 100644 --- a/src/jvm/storm/starter/ExclamationTopology.java +++ b/src/jvm/storm/starter/ExclamationTopology.java @@ -15,7 +15,9 @@ import backtype.storm.utils.Utils; import java.util.Map; - +/** + * This is a basic example of a Storm topology. + */ public class ExclamationTopology { public static class ExclamationBolt implements IRichBolt { diff --git a/src/jvm/storm/starter/ReachTopology.java b/src/jvm/storm/starter/ReachTopology.java new file mode 100644 index 0000000000..b914948077 --- /dev/null +++ b/src/jvm/storm/starter/ReachTopology.java @@ -0,0 +1,226 @@ +package storm.starter; + +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.LocalDRPC; +import backtype.storm.StormSubmitter; +import backtype.storm.drpc.CoordinatedBolt.FinishedCallback; +import backtype.storm.drpc.LinearDRPCTopologyBuilder; +import backtype.storm.task.OutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.BasicOutputCollector; +import backtype.storm.topology.IBasicBolt; +import backtype.storm.topology.IRichBolt; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * This is a good example of doing complex Distributed RPC on top of Storm. This + * program creates a topology that can compute the reach for any URL on Twitter + * in realtime by parallelizing the whole computation. + * + * Reach is the number of unique people exposed to a URL on Twitter. To compute reach, + * you have to get all the people who tweeted the URL, get all the followers of all those people, + * unique that set of followers, and then count the unique set. It's an intense computation + * that can involve thousands of database calls and tens of millions of follower records. + * + * This Storm topology does every piece of that computation in parallel, turning what would be a + * computation that takes minutes on a single machine into one that takes just a couple seconds. + * + * For the purposes of demonstration, this topology replaces the use of actual DBs with + * in-memory hashmaps. + * + * See https://github.com/nathanmarz/storm/wiki/Distributed-RPC for more information on Distributed RPC. + */ +public class ReachTopology { + public static Map> TWEETERS_DB = new HashMap>() {{ + put("foo.com/blog/1", Arrays.asList("sally", "bob", "tim", "george", "nathan")); + put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally", "nathan")); + put("tech.backtype.com/blog/123", Arrays.asList("tim", "mike", "john")); + }}; + + public static Map> FOLLOWERS_DB = new HashMap>() {{ + put("sally", Arrays.asList("bob", "tim", "alice", "adam", "jim", "chris", "jai")); + put("bob", Arrays.asList("sally", "nathan", "jim", "mary", "david", "vivian")); + put("tim", Arrays.asList("alex")); + put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily", "jordan")); + put("adam", Arrays.asList("david", "carissa")); + put("mike", Arrays.asList("john", "bob")); + put("john", Arrays.asList("alice", "nathan", "jim", "mike", "bob")); + }}; + + public static class GetTweeters implements IBasicBolt { + @Override + public void prepare(Map conf, TopologyContext context) { + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + Object id = tuple.getValue(0); + String url = tuple.getString(1); + List tweeters = TWEETERS_DB.get(url); + if(tweeters!=null) { + for(String tweeter: tweeters) { + collector.emit(new Values(id, tweeter)); + } + } + } + + @Override + public void cleanup() { + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "tweeter")); + } + } + + public static class GetFollowers implements IBasicBolt { + @Override + public void prepare(Map conf, TopologyContext context) { + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + Object id = tuple.getValue(0); + String tweeter = tuple.getString(1); + List followers = FOLLOWERS_DB.get(tweeter); + if(followers!=null) { + for(String follower: followers) { + collector.emit(new Values(id, follower)); + } + } + } + + @Override + public void cleanup() { + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "follower")); + } + } + + public static class PartialUniquer implements IRichBolt, FinishedCallback { + OutputCollector _collector; + Map> _sets = new HashMap>(); + + @Override + public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + _collector = collector; + } + + @Override + public void execute(Tuple tuple) { + Object id = tuple.getValue(0); + Set curr = _sets.get(id); + if(curr==null) { + curr = new HashSet(); + _sets.put(id, curr); + } + curr.add(tuple.getString(1)); + _collector.ack(tuple); + } + + @Override + public void cleanup() { + } + + @Override + public void finishedId(Object id) { + Set curr = _sets.remove(id); + int count; + if(curr!=null) { + count = curr.size(); + } else { + count = 0; + } + _collector.emit(new Values(id, count)); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "partial-count")); + } + } + + public static class CountAggregator implements IRichBolt, FinishedCallback { + Map _counts = new HashMap(); + OutputCollector _collector; + + @Override + public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + _collector = collector; + } + + @Override + public void execute(Tuple tuple) { + Object id = tuple.getValue(0); + int partial = tuple.getInteger(1); + + Integer curr = _counts.get(id); + if(curr==null) curr = 0; + _counts.put(id, curr + partial); + _collector.ack(tuple); + } + + @Override + public void cleanup() { + } + + @Override + public void finishedId(Object id) { + Integer reach = _counts.get(id); + if(reach==null) reach = 0; + _collector.emit(new Values(id, reach)); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "reach")); + } + + } + + public static void main(String[] args) throws Exception { + LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("reach"); + builder.addBolt(new GetTweeters(), 3); + builder.addBolt(new GetFollowers(), 12) + .shuffleGrouping(); + builder.addBolt(new PartialUniquer(), 6) + .fieldsGrouping(new Fields("id", "follower")); + builder.addBolt(new CountAggregator(), 2) + .fieldsGrouping(new Fields("id")); + + + Config conf = new Config(); + + if(args==null || args.length==0) { + conf.setMaxTaskParallelism(3); + LocalDRPC drpc = new LocalDRPC(); + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology("reach-drpc", conf, builder.createLocalTopology(drpc)); + + String[] urlsToTry = new String[] { "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com"}; + for(String url: urlsToTry) { + System.out.println("Reach of " + url + ": " + drpc.execute("reach", url)); + } + + cluster.shutdown(); + drpc.shutdown(); + } else { + conf.setNumWorkers(6); + StormSubmitter.submitTopology(args[0], conf, builder.createRemoteTopology()); + } + } +} diff --git a/src/jvm/storm/starter/RollingTopWords.java b/src/jvm/storm/starter/RollingTopWords.java index c39a918ad7..7145b06807 100644 --- a/src/jvm/storm/starter/RollingTopWords.java +++ b/src/jvm/storm/starter/RollingTopWords.java @@ -9,6 +9,13 @@ import storm.starter.bolt.RankObjects; import storm.starter.bolt.RollingCountObjects; + +/** + * This topology does a continuous computation of the top N words that the topology has seen + * in terms of cardinality. The top N computation is done in a completely scalable way, and + * a similar approach could be used to compute things like trending topics or trending images + * on Twitter. + */ public class RollingTopWords { public static void main(String[] args) throws Exception { diff --git a/src/jvm/storm/starter/WordCountTopology.java b/src/jvm/storm/starter/WordCountTopology.java index 400275c6d5..4d08287043 100644 --- a/src/jvm/storm/starter/WordCountTopology.java +++ b/src/jvm/storm/starter/WordCountTopology.java @@ -17,7 +17,9 @@ import java.util.HashMap; import java.util.Map; - +/** + * This topology demonstrates Storm's stream groupings and multilang capabilities. + */ public class WordCountTopology { public static class SplitSentence extends ShellBolt implements IRichBolt { From b4d90cbc054b4701648555d141ea5f9692f5261a Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sun, 23 Oct 2011 15:32:52 -0700 Subject: [PATCH 015/349] cleanup a topology --- src/jvm/storm/starter/BasicDRPCTopology.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/jvm/storm/starter/BasicDRPCTopology.java b/src/jvm/storm/starter/BasicDRPCTopology.java index fcd6ee575d..718add223e 100644 --- a/src/jvm/storm/starter/BasicDRPCTopology.java +++ b/src/jvm/storm/starter/BasicDRPCTopology.java @@ -48,9 +48,8 @@ public static void main(String[] args) throws Exception { builder.addBolt(new ExclaimBolt(), 3); Config conf = new Config(); - conf.setDebug(true); - if(args.length==0) { + if(args==null || args.length==0) { LocalDRPC drpc = new LocalDRPC(); LocalCluster cluster = new LocalCluster(); @@ -60,6 +59,9 @@ public static void main(String[] args) throws Exception { System.out.println("Result for \"" + word + "\": " + drpc.execute("exclamation", word)); } + + cluster.shutdown(); + drpc.shutdown(); } else { conf.setNumWorkers(3); StormSubmitter.submitTopology(args[0], conf, builder.createRemoteTopology()); From 6b57766422be5eae1395e589780748351dd37991 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sun, 23 Oct 2011 15:47:19 -0700 Subject: [PATCH 016/349] fix TwitterSampleSpout by relying on Java serialization for Status objects --- src/jvm/storm/starter/PrintSampleStream.java | 4 ++- src/jvm/storm/starter/bolt/PrinterBolt.java | 30 +++++++++++++++++++ .../starter/spout/TwitterSampleSpout.java | 11 ++++--- 3 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 src/jvm/storm/starter/bolt/PrinterBolt.java diff --git a/src/jvm/storm/starter/PrintSampleStream.java b/src/jvm/storm/starter/PrintSampleStream.java index 8a8445a255..0b2876f95a 100644 --- a/src/jvm/storm/starter/PrintSampleStream.java +++ b/src/jvm/storm/starter/PrintSampleStream.java @@ -5,6 +5,7 @@ import backtype.storm.LocalCluster; import backtype.storm.topology.TopologyBuilder; import backtype.storm.utils.Utils; +import storm.starter.bolt.PrinterBolt; public class PrintSampleStream { @@ -14,10 +15,11 @@ public static void main(String[] args) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(1, new TwitterSampleSpout(username, pwd)); + builder.setBolt(2, new PrinterBolt()) + .shuffleGrouping(1); Config conf = new Config(); - conf.setDebug(true); LocalCluster cluster = new LocalCluster(); diff --git a/src/jvm/storm/starter/bolt/PrinterBolt.java b/src/jvm/storm/starter/bolt/PrinterBolt.java new file mode 100644 index 0000000000..66d6262f57 --- /dev/null +++ b/src/jvm/storm/starter/bolt/PrinterBolt.java @@ -0,0 +1,30 @@ +package storm.starter.bolt; + +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.BasicOutputCollector; +import backtype.storm.topology.IBasicBolt; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.tuple.Tuple; +import java.util.Map; + + +public class PrinterBolt implements IBasicBolt { + + @Override + public void prepare(Map conf, TopologyContext context) { + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + System.out.println(tuple); + } + + @Override + public void cleanup() { + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer ofd) { + } + +} diff --git a/src/jvm/storm/starter/spout/TwitterSampleSpout.java b/src/jvm/storm/starter/spout/TwitterSampleSpout.java index 993e01de43..172b1a47d5 100644 --- a/src/jvm/storm/starter/spout/TwitterSampleSpout.java +++ b/src/jvm/storm/starter/spout/TwitterSampleSpout.java @@ -15,11 +15,10 @@ import twitter4j.Status; import twitter4j.StatusDeletionNotice; import twitter4j.StatusListener; -import twitter4j.json.DataObjectFactory; public class TwitterSampleSpout implements IRichSpout { SpoutOutputCollector _collector; - LinkedBlockingQueue queue = null; + LinkedBlockingQueue queue = null; TwitterStream _twitterStream; String _username; String _pwd; @@ -37,13 +36,13 @@ public boolean isDistributed() { @Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { - queue = new LinkedBlockingQueue(1000); + queue = new LinkedBlockingQueue(1000); _collector = collector; StatusListener listener = new StatusListener() { @Override - public void onStatus(Status status) { - queue.offer(DataObjectFactory.getRawJSON(status)); + public void onStatus(Status status) { + queue.offer(status); } @Override @@ -71,7 +70,7 @@ public void onException(Exception e) { @Override public void nextTuple() { - String ret = queue.poll(); + Status ret = queue.poll(); if(ret==null) { Utils.sleep(50); } else { From 1891618e21b01da8de5d43e6b95beab86a3444ac Mon Sep 17 00:00:00 2001 From: Taylor Goetz Date: Wed, 2 Nov 2011 15:06:41 -0400 Subject: [PATCH 017/349] Added maven2 pom for users who may not want to use lein to build. --- m2-pom.xml | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 m2-pom.xml diff --git a/m2-pom.xml b/m2-pom.xml new file mode 100644 index 0000000000..79f7001449 --- /dev/null +++ b/m2-pom.xml @@ -0,0 +1,116 @@ + + 4.0.0 + + storm.starter + storm-starter + 0.0.1-SNAPSHOT + jar + + storm-starter + https://github.com/nathanmarz/storm-starter + + + UTF-8 + + + + github-releases + http://oss.sonatype.org/content/repositories/github-releases/ + + + clojars.org + http://clojars.org/repo + + + twitter4j + http://twitter4j.org/maven2 + + + + + + junit + junit + 3.8.1 + test + + + storm + storm + 0.5.4 + + + org.twitter4j + twitter4j-core + 2.2.5-SNAPSHOT + + + org.twitter4j + twitter4j-stream + 2.2.5-SNAPSHOT + + + org.clojure + clojure + 1.2.0 + + + org.clojure + clojure-contrib + 1.2.0 + + + com.googlecode.json-simple + json-simple + 1.1 + + + + + src/jvm + + + ${basedir}/multilang + + + + + com.theoryinpractise + clojure-maven-plugin + 1.3.8 + true + + + src/clj + + + + + compile + compile + + compile + + + + test + test + + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.6 + 1.6 + + + + + From 91550313b5288e9c059165e6ce8bad180fb764cb Mon Sep 17 00:00:00 2001 From: ptgoetz Date: Wed, 2 Nov 2011 17:16:04 -0400 Subject: [PATCH 018/349] Updated maven2 pom to create a storm-deployable jar that excludes storm dependencies. --- m2-pom.xml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/m2-pom.xml b/m2-pom.xml index 79f7001449..69bbb819ba 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -39,6 +39,8 @@ storm storm 0.5.4 + + provided org.twitter4j @@ -75,6 +77,35 @@ + + + maven-assembly-plugin + + + jar-with-dependencies + + + + + + + + + + make-assembly + package + + single + + + + + + com.theoryinpractise clojure-maven-plugin From 4edef6a0dcbe136820974aabda649132175a7dd5 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 3 Nov 2011 16:59:32 -0700 Subject: [PATCH 019/349] null check so easier to run topologies from clojure REPL --- src/jvm/storm/starter/ExclamationTopology.java | 2 +- src/jvm/storm/starter/WordCountTopology.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jvm/storm/starter/ExclamationTopology.java b/src/jvm/storm/starter/ExclamationTopology.java index c6080f63a3..de0f78b70d 100644 --- a/src/jvm/storm/starter/ExclamationTopology.java +++ b/src/jvm/storm/starter/ExclamationTopology.java @@ -58,7 +58,7 @@ public static void main(String[] args) throws Exception { Config conf = new Config(); conf.setDebug(true); - if(args.length > 0) { + if(args!=null && args.length > 0) { conf.setNumWorkers(3); StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); diff --git a/src/jvm/storm/starter/WordCountTopology.java b/src/jvm/storm/starter/WordCountTopology.java index 4d08287043..ad1ee247e7 100644 --- a/src/jvm/storm/starter/WordCountTopology.java +++ b/src/jvm/storm/starter/WordCountTopology.java @@ -75,7 +75,7 @@ public static void main(String[] args) throws Exception { conf.setDebug(true); - if(args.length > 0) { + if(args!=null && args.length > 0) { conf.setNumWorkers(3); StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); From 65bfaf3d045b3289fd3e08de6b85426f3c369c99 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sun, 6 Nov 2011 21:00:16 -0800 Subject: [PATCH 020/349] added example of doing single join --- src/jvm/storm/starter/SingleJoinExample.java | 47 ++++++++ .../storm/starter/bolt/SingleJoinBolt.java | 102 ++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 src/jvm/storm/starter/SingleJoinExample.java create mode 100644 src/jvm/storm/starter/bolt/SingleJoinBolt.java diff --git a/src/jvm/storm/starter/SingleJoinExample.java b/src/jvm/storm/starter/SingleJoinExample.java new file mode 100644 index 0000000000..c90e58aa45 --- /dev/null +++ b/src/jvm/storm/starter/SingleJoinExample.java @@ -0,0 +1,47 @@ +package storm.starter; + +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.testing.FeederSpout; +import backtype.storm.topology.TopologyBuilder; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Values; +import backtype.storm.utils.Utils; +import storm.starter.bolt.SingleJoinBolt; + +public class SingleJoinExample { + public static void main(String[] args) { + FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender")); + FeederSpout ageSpout = new FeederSpout(new Fields("id", "age")); + + TopologyBuilder builder = new TopologyBuilder(); + builder.setSpout(1, genderSpout); + builder.setSpout(2, ageSpout); + builder.setBolt(3, new SingleJoinBolt(new Fields("gender", "age"))) + .fieldsGrouping(1, new Fields("id")) + .fieldsGrouping(2, new Fields("id")); + + Config conf = new Config(); + conf.setDebug(true); + + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology("join-example", conf, builder.createTopology()); + + for(int i=0; i<10; i++) { + String gender; + if(i % 2 == 0) { + gender = "male"; + } else { + gender = "female"; + } + genderSpout.feed(new Values(i, gender)); + } + + for(int i=9; i>=0; i--) { + ageSpout.feed(new Values(i, i+20)); + } + + Utils.sleep(2000); + cluster.shutdown(); + } +} diff --git a/src/jvm/storm/starter/bolt/SingleJoinBolt.java b/src/jvm/storm/starter/bolt/SingleJoinBolt.java new file mode 100644 index 0000000000..df944e0737 --- /dev/null +++ b/src/jvm/storm/starter/bolt/SingleJoinBolt.java @@ -0,0 +1,102 @@ +package storm.starter.bolt; + +import backtype.storm.Config; +import backtype.storm.generated.GlobalStreamId; +import backtype.storm.task.OutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.IRichBolt; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.utils.TimeCacheMap; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class SingleJoinBolt implements IRichBolt { + OutputCollector _collector; + Fields _idFields; + Fields _outFields; + int _numSources; + TimeCacheMap, Map> _pending; + Map _fieldLocations; + + public SingleJoinBolt(Fields outFields) { + _outFields = outFields; + } + + @Override + public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + _fieldLocations = new HashMap(); + _collector = collector; + long timeout = (Long) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS); + _pending = new TimeCacheMap, Map>((int) timeout, new ExpireCallback()); + _numSources = context.getThisSources().size(); + Set idFields = null; + for(GlobalStreamId source: context.getThisSources().keySet()) { + Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId()); + Set setFields = new HashSet(fields.toList()); + if(idFields==null) idFields = setFields; + else idFields.retainAll(setFields); + + for(String outfield: _outFields) { + for(String sourcefield: fields) { + if(outfield.equals(sourcefield)) { + _fieldLocations.put(outfield, source); + } + } + } + } + _idFields = new Fields(new ArrayList(idFields)); + + if(_fieldLocations.size()!=_outFields.size()) { + throw new RuntimeException("Cannot find all outfields among sources"); + } + } + + @Override + public void execute(Tuple tuple) { + List id = tuple.select(_idFields); + GlobalStreamId streamId = new GlobalStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId()); + if(!_pending.containsKey(id)) { + _pending.put(id, new HashMap()); + } + Map parts = _pending.get(id); + if(parts.containsKey(streamId)) throw new RuntimeException("Received same side of single join twice"); + parts.put(streamId, tuple); + if(parts.size()==_numSources) { + _pending.remove(id); + List joinResult = new ArrayList(); + for(String outField: _outFields) { + GlobalStreamId loc = _fieldLocations.get(outField); + joinResult.add(parts.get(loc).getValueByField(outField)); + } + _collector.emit(new ArrayList(parts.values()), joinResult); + + for(Tuple part: parts.values()) { + _collector.ack(part); + } + } + } + + @Override + public void cleanup() { + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(_outFields); + } + + private class ExpireCallback implements TimeCacheMap.ExpiredCallback, Map> { + @Override + public void expire(List id, Map tuples) { + for(Tuple tuple: tuples.values()) { + _collector.fail(tuple); + } + } + } +} From 82b347c3168d2cc0123a43f9690a425f605f966d Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sun, 6 Nov 2011 21:00:23 -0800 Subject: [PATCH 021/349] update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5f30b63ed0..6206669be2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ lib/ *.pyc .classpath .project +*.jar From e7adbdb587e04ce71542473062dbcad9d888e8ed Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sun, 6 Nov 2011 22:46:11 -0800 Subject: [PATCH 022/349] refresh README --- README | 1 - README.markdown | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) delete mode 100644 README create mode 100644 README.markdown diff --git a/README b/README deleted file mode 100644 index ed74288d02..0000000000 --- a/README +++ /dev/null @@ -1 +0,0 @@ -Example Storm topologies diff --git a/README.markdown b/README.markdown new file mode 100644 index 0000000000..96be64f64d --- /dev/null +++ b/README.markdown @@ -0,0 +1,22 @@ +# Example Storm topologies + +storm-starter contains a variety of examples of using Storm. If this is your first time checking out Storm, check out these topologies first: + +1. ExclamationTopology: Basic topology written in all Java +2. WordCountTopology: Basic topology that makes use of multilang by implementing one bolt in Python +3. ReachTopology: Example of complex DRPC on top of Storm + +More information about Storm can be found on the [project page](http://github.com/nathanmarz/storm) + +## Running an example with Leiningen + +lein deps +lein compile +java -cp `lein classpath` storm.starter.ExclamationTopology + + +## Maven + +Maven is an alternative to Leiningen. storm-starter contains m2-pom.xml which can be used with Maven using the -f option, e.g.: + +mvn -f m2-pom.xml package \ No newline at end of file From 1f5cb7fe76ac3a56e9b0f6f6f5b2df0025003303 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sun, 6 Nov 2011 22:46:49 -0800 Subject: [PATCH 023/349] fix README --- README.markdown | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 96be64f64d..20598b365c 100644 --- a/README.markdown +++ b/README.markdown @@ -10,10 +10,11 @@ More information about Storm can be found on the [project page](http://github.co ## Running an example with Leiningen +``` lein deps lein compile java -cp `lein classpath` storm.starter.ExclamationTopology - +``` ## Maven From de9955f5f31ff6a9cd17444075a26f5565b65368 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sun, 6 Nov 2011 22:47:19 -0800 Subject: [PATCH 024/349] fix README --- README.markdown | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 20598b365c..e88d8e57f7 100644 --- a/README.markdown +++ b/README.markdown @@ -20,4 +20,6 @@ java -cp `lein classpath` storm.starter.ExclamationTopology Maven is an alternative to Leiningen. storm-starter contains m2-pom.xml which can be used with Maven using the -f option, e.g.: -mvn -f m2-pom.xml package \ No newline at end of file +``` +mvn -f m2-pom.xml package +``` \ No newline at end of file From 3b815c3aa5d8b3500c9c0ff84ee41e9f6b748c56 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sun, 6 Nov 2011 22:47:35 -0800 Subject: [PATCH 025/349] fix README --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index e88d8e57f7..08a235df5a 100644 --- a/README.markdown +++ b/README.markdown @@ -6,7 +6,7 @@ storm-starter contains a variety of examples of using Storm. If this is your fir 2. WordCountTopology: Basic topology that makes use of multilang by implementing one bolt in Python 3. ReachTopology: Example of complex DRPC on top of Storm -More information about Storm can be found on the [project page](http://github.com/nathanmarz/storm) +More information about Storm can be found on the [project page](http://github.com/nathanmarz/storm). ## Running an example with Leiningen From e31f0452e1e3b091a69c6c3a88cd7bf04ef4c189 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 30 Nov 2011 21:18:47 -0800 Subject: [PATCH 026/349] update storm-starter for storm 0.6.0 --- m2-pom.xml | 2 +- project.clj | 2 +- src/clj/storm/starter/clj/word_count.clj | 22 +++++++++---------- .../storm/starter/ExclamationTopology.java | 10 ++++----- src/jvm/storm/starter/PrintSampleStream.java | 6 ++--- src/jvm/storm/starter/ReachTopology.java | 18 ++++++++++----- src/jvm/storm/starter/RollingTopWords.java | 16 +++++++------- src/jvm/storm/starter/SingleJoinExample.java | 10 ++++----- src/jvm/storm/starter/WordCountTopology.java | 10 ++++----- 9 files changed, 51 insertions(+), 45 deletions(-) diff --git a/m2-pom.xml b/m2-pom.xml index 69bbb819ba..c2039e03ca 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -38,7 +38,7 @@ storm storm - 0.5.4 + 0.6.0 provided diff --git a/project.clj b/project.clj index fafdf4a608..85c51e788b 100644 --- a/project.clj +++ b/project.clj @@ -16,6 +16,6 @@ [com.googlecode.json-simple/json-simple "1.1"] ] - :dev-dependencies [[storm "0.5.4"] + :dev-dependencies [[storm "0.6.0"] ]) diff --git a/src/clj/storm/starter/clj/word_count.clj b/src/clj/storm/starter/clj/word_count.clj index 5c0a47291d..dc4db280a4 100644 --- a/src/clj/storm/starter/clj/word_count.clj +++ b/src/clj/storm/starter/clj/word_count.clj @@ -46,17 +46,17 @@ (defn mk-topology [] (topology - {1 (spout-spec sentence-spout) - 2 (spout-spec (sentence-spout-parameterized - ["the cat jumped over the door" - "greetings from a faraway land"]) - :p 2)} - {3 (bolt-spec {1 :shuffle 2 :shuffle} - split-sentence - :p 5) - 4 (bolt-spec {3 ["word"]} - word-count - :p 6)})) + {"1" (spout-spec sentence-spout) + "2" (spout-spec (sentence-spout-parameterized + ["the cat jumped over the door" + "greetings from a faraway land"]) + :p 2)} + {"3" (bolt-spec {"1" :shuffle "2" :shuffle} + split-sentence + :p 5) + "4" (bolt-spec {"3" ["word"]} + word-count + :p 6)})) (defn run-local! [] (let [cluster (LocalCluster.)] diff --git a/src/jvm/storm/starter/ExclamationTopology.java b/src/jvm/storm/starter/ExclamationTopology.java index de0f78b70d..8a5b381c24 100644 --- a/src/jvm/storm/starter/ExclamationTopology.java +++ b/src/jvm/storm/starter/ExclamationTopology.java @@ -49,11 +49,11 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); - builder.setSpout(1, new TestWordSpout(), 10); - builder.setBolt(2, new ExclamationBolt(), 3) - .shuffleGrouping(1); - builder.setBolt(3, new ExclamationBolt(), 2) - .shuffleGrouping(2); + builder.setSpout("word", new TestWordSpout(), 10); + builder.setBolt("exclaim1", new ExclamationBolt(), 3) + .shuffleGrouping("word"); + builder.setBolt("exclaim2", new ExclamationBolt(), 2) + .shuffleGrouping("exclaim1"); Config conf = new Config(); conf.setDebug(true); diff --git a/src/jvm/storm/starter/PrintSampleStream.java b/src/jvm/storm/starter/PrintSampleStream.java index 0b2876f95a..b3524b58d6 100644 --- a/src/jvm/storm/starter/PrintSampleStream.java +++ b/src/jvm/storm/starter/PrintSampleStream.java @@ -14,9 +14,9 @@ public static void main(String[] args) { String pwd = args[1]; TopologyBuilder builder = new TopologyBuilder(); - builder.setSpout(1, new TwitterSampleSpout(username, pwd)); - builder.setBolt(2, new PrinterBolt()) - .shuffleGrouping(1); + builder.setSpout("spout", new TwitterSampleSpout(username, pwd)); + builder.setBolt("print", new PrinterBolt()) + .shuffleGrouping("spout"); Config conf = new Config(); diff --git a/src/jvm/storm/starter/ReachTopology.java b/src/jvm/storm/starter/ReachTopology.java index b914948077..4d4ab287de 100644 --- a/src/jvm/storm/starter/ReachTopology.java +++ b/src/jvm/storm/starter/ReachTopology.java @@ -192,18 +192,24 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } - public static void main(String[] args) throws Exception { + public static LinearDRPCTopologyBuilder construct() { LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("reach"); - builder.addBolt(new GetTweeters(), 3); + builder.addBolt(new GetTweeters(), 4); builder.addBolt(new GetFollowers(), 12) - .shuffleGrouping(); + .shuffleGrouping(); builder.addBolt(new PartialUniquer(), 6) - .fieldsGrouping(new Fields("id", "follower")); - builder.addBolt(new CountAggregator(), 2) - .fieldsGrouping(new Fields("id")); + .fieldsGrouping(new Fields("id", "follower")); + builder.addBolt(new CountAggregator(), 3) + .fieldsGrouping(new Fields("id")); + return builder; + } + + public static void main(String[] args) throws Exception { + LinearDRPCTopologyBuilder builder = construct(); Config conf = new Config(); + conf.setDebug(true); if(args==null || args.length==0) { conf.setMaxTaskParallelism(3); diff --git a/src/jvm/storm/starter/RollingTopWords.java b/src/jvm/storm/starter/RollingTopWords.java index 7145b06807..b105c5a700 100644 --- a/src/jvm/storm/starter/RollingTopWords.java +++ b/src/jvm/storm/starter/RollingTopWords.java @@ -24,14 +24,14 @@ public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); - builder.setSpout(1, new TestWordSpout(), 5); - - builder.setBolt(2, new RollingCountObjects(60, 10), 4) - .fieldsGrouping(1, new Fields("word")); - builder.setBolt(3, new RankObjects(TOP_N), 4) - .fieldsGrouping(2, new Fields("obj")); - builder.setBolt(4, new MergeObjects(TOP_N)) - .globalGrouping(3); + builder.setSpout("word", new TestWordSpout(), 5); + + builder.setBolt("count", new RollingCountObjects(60, 10), 4) + .fieldsGrouping("word", new Fields("word")); + builder.setBolt("rank", new RankObjects(TOP_N), 4) + .fieldsGrouping("count", new Fields("obj")); + builder.setBolt("merge", new MergeObjects(TOP_N)) + .globalGrouping("rank"); diff --git a/src/jvm/storm/starter/SingleJoinExample.java b/src/jvm/storm/starter/SingleJoinExample.java index c90e58aa45..62d332ea80 100644 --- a/src/jvm/storm/starter/SingleJoinExample.java +++ b/src/jvm/storm/starter/SingleJoinExample.java @@ -15,11 +15,11 @@ public static void main(String[] args) { FeederSpout ageSpout = new FeederSpout(new Fields("id", "age")); TopologyBuilder builder = new TopologyBuilder(); - builder.setSpout(1, genderSpout); - builder.setSpout(2, ageSpout); - builder.setBolt(3, new SingleJoinBolt(new Fields("gender", "age"))) - .fieldsGrouping(1, new Fields("id")) - .fieldsGrouping(2, new Fields("id")); + builder.setSpout("gender", genderSpout); + builder.setSpout("age", ageSpout); + builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age"))) + .fieldsGrouping("gender", new Fields("id")) + .fieldsGrouping("age", new Fields("id")); Config conf = new Config(); conf.setDebug(true); diff --git a/src/jvm/storm/starter/WordCountTopology.java b/src/jvm/storm/starter/WordCountTopology.java index ad1ee247e7..9ed988de0d 100644 --- a/src/jvm/storm/starter/WordCountTopology.java +++ b/src/jvm/storm/starter/WordCountTopology.java @@ -64,12 +64,12 @@ public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); - builder.setSpout(1, new RandomSentenceSpout(), 5); + builder.setSpout("spout", new RandomSentenceSpout(), 5); - builder.setBolt(2, new SplitSentence(), 8) - .shuffleGrouping(1); - builder.setBolt(3, new WordCount(), 12) - .fieldsGrouping(2, new Fields("word")); + builder.setBolt("split", new SplitSentence(), 8) + .shuffleGrouping("spout"); + builder.setBolt("count", new WordCount(), 12) + .fieldsGrouping("split", new Fields("word")); Config conf = new Config(); conf.setDebug(true); From b0805c8504f4a1ff31ab5c7a919d1c8ff26cec81 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 30 Nov 2011 21:21:19 -0800 Subject: [PATCH 027/349] remove debug from reachtopology --- src/jvm/storm/starter/ReachTopology.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/jvm/storm/starter/ReachTopology.java b/src/jvm/storm/starter/ReachTopology.java index 4d4ab287de..82aa772379 100644 --- a/src/jvm/storm/starter/ReachTopology.java +++ b/src/jvm/storm/starter/ReachTopology.java @@ -209,7 +209,6 @@ public static void main(String[] args) throws Exception { Config conf = new Config(); - conf.setDebug(true); if(args==null || args.length==0) { conf.setMaxTaskParallelism(3); From 98915c3fe9fd71f8009831d65c3e539503b8a978 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 1 Dec 2011 17:09:27 -0800 Subject: [PATCH 028/349] improve rolling top words to take advantage of improved serialization --- project.clj | 1 - src/jvm/storm/starter/bolt/MergeObjects.java | 8 +++----- src/jvm/storm/starter/bolt/RankObjects.java | 3 +-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/project.clj b/project.clj index 85c51e788b..de52d5fc3c 100644 --- a/project.clj +++ b/project.clj @@ -13,7 +13,6 @@ [org.clojure/clojure-contrib "1.2.0"] [org.twitter4j/twitter4j-core "2.2.5-SNAPSHOT"] [org.twitter4j/twitter4j-stream "2.2.5-SNAPSHOT"] - [com.googlecode.json-simple/json-simple "1.1"] ] :dev-dependencies [[storm "0.6.0"] diff --git a/src/jvm/storm/starter/bolt/MergeObjects.java b/src/jvm/storm/starter/bolt/MergeObjects.java index aa54d6cb13..ee45f57526 100644 --- a/src/jvm/storm/starter/bolt/MergeObjects.java +++ b/src/jvm/storm/starter/bolt/MergeObjects.java @@ -13,7 +13,6 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import org.json.simple.JSONValue; public class MergeObjects implements IBasicBolt { @@ -66,7 +65,7 @@ public void prepare(Map stormConf, TopologyContext context) { public void execute(Tuple tuple, BasicOutputCollector collector) { - List merging = (List) JSONValue.parse(tuple.getString(0)); + List merging = (List) tuple.getValue(0); for(List pair : merging) { Integer existingIndex = _find(pair.get(0)); @@ -93,9 +92,8 @@ public int compare(List o1, List o2) { long currentTime = System.currentTimeMillis(); if(_lastTime==null || currentTime >= _lastTime + 2000) { - String fullRankings = JSONValue.toJSONString(_rankings); - collector.emit(new Values(fullRankings)); - LOG.info("Rankings: " + fullRankings); + collector.emit(new Values(_rankings)); + LOG.info("Rankings: " + _rankings); _lastTime = currentTime; } } diff --git a/src/jvm/storm/starter/bolt/RankObjects.java b/src/jvm/storm/starter/bolt/RankObjects.java index 68ab325fc4..20f43ffe4d 100644 --- a/src/jvm/storm/starter/bolt/RankObjects.java +++ b/src/jvm/storm/starter/bolt/RankObjects.java @@ -12,7 +12,6 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import org.json.simple.JSONValue; public class RankObjects implements IBasicBolt { @@ -90,7 +89,7 @@ public int compare(List o1, List o2) { long currentTime = System.currentTimeMillis(); if(_lastTime==null || currentTime >= _lastTime + 2000) { - collector.emit(new Values(JSONValue.toJSONString(_rankings))); + collector.emit(new Values(_rankings)); _lastTime = currentTime; } } From 4186391cf91cb4ecc492eea50c8e3cc1cc37c828 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 1 Dec 2011 19:51:55 -0800 Subject: [PATCH 029/349] formatting --- src/jvm/storm/starter/bolt/MergeObjects.java | 19 +--------------- src/jvm/storm/starter/bolt/RankObjects.java | 24 +------------------- 2 files changed, 2 insertions(+), 41 deletions(-) diff --git a/src/jvm/storm/starter/bolt/MergeObjects.java b/src/jvm/storm/starter/bolt/MergeObjects.java index ee45f57526..e40204cec4 100644 --- a/src/jvm/storm/starter/bolt/MergeObjects.java +++ b/src/jvm/storm/starter/bolt/MergeObjects.java @@ -14,7 +14,6 @@ import java.util.List; import java.util.Map; - public class MergeObjects implements IBasicBolt { public static Logger LOG = Logger.getLogger(MergeObjects.class); @@ -26,12 +25,9 @@ public MergeObjects(int n) { _count = n; } - private int _compare(List one, List two) { - long valueOne = (Long) one.get(1); long valueTwo = (Long) two.get(1); - long delta = valueTwo - valueOne; if(delta > 0) { return 1; @@ -40,21 +36,16 @@ private int _compare(List one, List two) { } else { return 0; } - - } //end compare + } private Integer _find(Object tag) { for(int i = 0; i < _rankings.size(); ++i) { - Object cur = _rankings.get(i).get(0); if (cur.equals(tag)) { return i; } - } - return null; - } @@ -63,18 +54,13 @@ public void prepare(Map stormConf, TopologyContext context) { } public void execute(Tuple tuple, BasicOutputCollector collector) { - - List merging = (List) tuple.getValue(0); for(List pair : merging) { - Integer existingIndex = _find(pair.get(0)); if (null != existingIndex) { _rankings.set(existingIndex, pair); } else { - _rankings.add(pair); - } Collections.sort(_rankings, new Comparator() { @@ -83,11 +69,9 @@ public int compare(List o1, List o2) { } }); - if (_rankings.size() > _count) { _rankings.subList(_count, _rankings.size()).clear(); } - } long currentTime = System.currentTimeMillis(); @@ -104,5 +88,4 @@ public void cleanup() { public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("list")); } - } diff --git a/src/jvm/storm/starter/bolt/RankObjects.java b/src/jvm/storm/starter/bolt/RankObjects.java index 20f43ffe4d..e5b3cafacd 100644 --- a/src/jvm/storm/starter/bolt/RankObjects.java +++ b/src/jvm/storm/starter/bolt/RankObjects.java @@ -15,7 +15,6 @@ public class RankObjects implements IBasicBolt { - List _rankings = new ArrayList(); int _count; @@ -25,12 +24,9 @@ public RankObjects(int n) { _count = n; } - private int _compare(List one, List two) { - long valueOne = (Long) one.get(1); long valueTwo = (Long) two.get(1); - long delta = valueTwo - valueOne; if(delta > 0) { return 1; @@ -39,54 +35,37 @@ private int _compare(List one, List two) { } else { return 0; } - - } //end compare + } private Integer _find(Object tag) { for(int i = 0; i < _rankings.size(); ++i) { - Object cur = _rankings.get(i).get(0); if (cur.equals(tag)) { return i; } - } - return null; - } public void prepare(Map stormConf, TopologyContext context) { - } public void execute(Tuple tuple, BasicOutputCollector collector) { - Object tag = tuple.getValue(0); - - Integer existingIndex = _find(tag); if (null != existingIndex) { _rankings.set(existingIndex, tuple.getValues()); } else { - _rankings.add(tuple.getValues()); - - } - - Collections.sort(_rankings, new Comparator() { public int compare(List o1, List o2) { return _compare(o1, o2); } }); - - if (_rankings.size() > _count) { _rankings.remove(_count); } - long currentTime = System.currentTimeMillis(); if(_lastTime==null || currentTime >= _lastTime + 2000) { collector.emit(new Values(_rankings)); @@ -100,5 +79,4 @@ public void cleanup() { public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("list")); } - } From 36395919fb4aecc38d0f1fb75ea066175df4f7b1 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Tue, 10 Jan 2012 23:27:52 -0800 Subject: [PATCH 030/349] make outputs of rankobjects and mergeobjects immutable --- src/jvm/storm/starter/bolt/MergeObjects.java | 2 +- src/jvm/storm/starter/bolt/RankObjects.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jvm/storm/starter/bolt/MergeObjects.java b/src/jvm/storm/starter/bolt/MergeObjects.java index e40204cec4..97bdaf8f77 100644 --- a/src/jvm/storm/starter/bolt/MergeObjects.java +++ b/src/jvm/storm/starter/bolt/MergeObjects.java @@ -76,7 +76,7 @@ public int compare(List o1, List o2) { long currentTime = System.currentTimeMillis(); if(_lastTime==null || currentTime >= _lastTime + 2000) { - collector.emit(new Values(_rankings)); + collector.emit(new Values(new ArrayList(_rankings))); LOG.info("Rankings: " + _rankings); _lastTime = currentTime; } diff --git a/src/jvm/storm/starter/bolt/RankObjects.java b/src/jvm/storm/starter/bolt/RankObjects.java index e5b3cafacd..e6c9e501f0 100644 --- a/src/jvm/storm/starter/bolt/RankObjects.java +++ b/src/jvm/storm/starter/bolt/RankObjects.java @@ -68,7 +68,7 @@ public int compare(List o1, List o2) { } long currentTime = System.currentTimeMillis(); if(_lastTime==null || currentTime >= _lastTime + 2000) { - collector.emit(new Values(_rankings)); + collector.emit(new Values(new ArrayList(_rankings))); _lastTime = currentTime; } } From d46e82e92538c0625f117c91f8fb85c7df80bb84 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 18 Jan 2012 02:08:22 -0800 Subject: [PATCH 031/349] untested transactionalspout implementation for kafka 0.6 --- project.clj | 8 + src/jvm/storm/kafka/BatchMeta.java | 6 + .../storm/kafka/TransactionalKafkaSpout.java | 160 ++++++++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 project.clj create mode 100644 src/jvm/storm/kafka/BatchMeta.java create mode 100644 src/jvm/storm/kafka/TransactionalKafkaSpout.java diff --git a/project.clj b/project.clj new file mode 100644 index 0000000000..c6923a942b --- /dev/null +++ b/project.clj @@ -0,0 +1,8 @@ +(defproject storm/storm-kafka "0.7.0-SNAPSHOT" + :source-path "src/clj" + :java-source-path "src/jvm" + :javac-options {:debug "true" :fork "true"} + :dependencies [[storm/kafka "0.6.0"]] + :dev-dependencies [[storm "0.7.0-SNAPSHOT"]] + :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] +) diff --git a/src/jvm/storm/kafka/BatchMeta.java b/src/jvm/storm/kafka/BatchMeta.java new file mode 100644 index 0000000000..4afbe8aeec --- /dev/null +++ b/src/jvm/storm/kafka/BatchMeta.java @@ -0,0 +1,6 @@ +package storm.kafka; + +public class BatchMeta { + long offset; + long nextOffset; +} diff --git a/src/jvm/storm/kafka/TransactionalKafkaSpout.java b/src/jvm/storm/kafka/TransactionalKafkaSpout.java new file mode 100644 index 0000000000..a9cbec1ee8 --- /dev/null +++ b/src/jvm/storm/kafka/TransactionalKafkaSpout.java @@ -0,0 +1,160 @@ +package storm.kafka; + +import backtype.storm.spout.RawScheme; +import backtype.storm.spout.Scheme; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BasePartitionedTransactionalSpout; +import backtype.storm.transactional.TransactionAttempt; +import backtype.storm.transactional.TransactionalOutputCollector; +import backtype.storm.transactional.partitioned.IPartitionedTransactionalSpout; +import backtype.storm.tuple.Fields; +import backtype.storm.utils.Utils; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import kafka.api.FetchRequest; +import kafka.javaapi.consumer.SimpleConsumer; +import kafka.javaapi.message.ByteBufferMessageSet; +import kafka.message.Message; +import kafka.message.MessageSet; + + +public class TransactionalKafkaSpout extends BasePartitionedTransactionalSpout { + public static final String ATTEMPT_FIELD = TransactionalKafkaSpout.class.getCanonicalName() + "/attempt"; + + public static class Config implements Serializable { + public List hosts; + public int port = 9092; + public int partitionsPerHost; + public int fetchSizeBytes = 1024*1024; + public int socketTimeoutMs = 10000; + public int bufferSizeBytes = 1024*1024; + public Scheme scheme = new RawScheme(); + public String topic; + + public Config(List hosts, int partitionsPerHost, String topic) { + this.hosts = hosts; + this.partitionsPerHost = partitionsPerHost; + this.topic = topic; + } + } + + Config _config; + + public TransactionalKafkaSpout(Config config) { + _config = config; + } + + class Coordinator implements IPartitionedTransactionalSpout.Coordinator { + @Override + public int numPartitions() { + return computeNumPartitions(); + } + + @Override + public void close() { + } + } + + class Emitter implements IPartitionedTransactionalSpout.Emitter { + Map _kafka = new HashMap(); + + @Override + public BatchMeta emitPartitionBatchNew(TransactionAttempt attempt, TransactionalOutputCollector collector, int partition, BatchMeta lastMeta) { + SimpleConsumer consumer = connect(partition); + + long offset = 0; + if(lastMeta!=null) { + offset = lastMeta.nextOffset; + } + + ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition % _config.partitionsPerHost, offset, _config.fetchSizeBytes)); + long endoffset = offset; + for(Message msg: msgs) { + emit(attempt, collector, msg); + endoffset += MessageSet.entrySize(msg); + } + BatchMeta newMeta = new BatchMeta(); + newMeta.offset = offset; + newMeta.nextOffset = endoffset; + return newMeta; + } + + @Override + public void emitPartitionBatch(TransactionAttempt attempt, TransactionalOutputCollector collector, int partition, BatchMeta meta) { + SimpleConsumer consumer = connect(partition); + + ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition % _config.partitionsPerHost, meta.offset, _config.fetchSizeBytes)); + long currOffset = meta.offset; + for(Message msg: msgs) { + if(currOffset == meta.nextOffset) break; + if(currOffset > meta.nextOffset) { + throw new RuntimeException("Error when re-emitting batch. overshot the end offset"); + } + emit(attempt, collector, msg); + currOffset += MessageSet.entrySize(msg); + + } + } + + private void emit(TransactionAttempt attempt, TransactionalOutputCollector collector, Message msg) { + List values = _config.scheme.deserialize(Utils.toByteArray(msg.payload())); + List toEmit = new ArrayList(); + toEmit.add(attempt); + toEmit.addAll(values); + collector.emit(toEmit); + } + + private SimpleConsumer connect(int partition) { + if(!_kafka.containsKey(partition)) { + int hostIndex = partition % _config.hosts.size(); + _kafka.put(partition, new SimpleConsumer(_config.hosts.get(hostIndex), _config.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); + } + return _kafka.get(partition); + } + + @Override + public void close() { + for(SimpleConsumer consumer: _kafka.values()) { + consumer.close(); + } + } + } + + + @Override + public IPartitionedTransactionalSpout.Coordinator getCoordinator(Map conf, TopologyContext context) { + return new Coordinator(); + } + + @Override + public IPartitionedTransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) { + return new Emitter(); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + List fields = new ArrayList(_config.scheme.getOutputFields().toList()); + fields.add(0, ATTEMPT_FIELD); + declarer.declare(new Fields(fields)); + } + + private int computeNumPartitions() { + return _config.hosts.size() * _config.partitionsPerHost; + } + + public static class BatchMeta { + long offset; + long nextOffset; + } + + @Override + public Map getComponentConfiguration() { + backtype.storm.Config conf = new backtype.storm.Config(); + conf.registerSerialization(BatchMeta.class); + return conf; + } +} \ No newline at end of file From 1c801a983b301ed72440ec8ca305906997558b0f Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 18 Jan 2012 02:24:30 -0800 Subject: [PATCH 032/349] add .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..ab43f1220e --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +classes/ +lib/ From 5b6c87fb1c31436cf4db1ba399471f66dd97b2d8 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 18 Jan 2012 02:24:50 -0800 Subject: [PATCH 033/349] remove redundant BatchMeta --- src/jvm/storm/kafka/TransactionalKafkaSpout.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/jvm/storm/kafka/TransactionalKafkaSpout.java b/src/jvm/storm/kafka/TransactionalKafkaSpout.java index a9cbec1ee8..4d9b636e6a 100644 --- a/src/jvm/storm/kafka/TransactionalKafkaSpout.java +++ b/src/jvm/storm/kafka/TransactionalKafkaSpout.java @@ -145,11 +145,6 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { private int computeNumPartitions() { return _config.hosts.size() * _config.partitionsPerHost; } - - public static class BatchMeta { - long offset; - long nextOffset; - } @Override public Map getComponentConfiguration() { From e12b26878d95ab02325baed7902f8c55cfb245ff Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 18 Jan 2012 03:28:37 -0800 Subject: [PATCH 034/349] exclude zookeeper client from storm-kafka since it's already available from storm --- project.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project.clj b/project.clj index c6923a942b..716b9f7b65 100644 --- a/project.clj +++ b/project.clj @@ -2,7 +2,8 @@ :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} - :dependencies [[storm/kafka "0.6.0"]] + :dependencies [[storm/kafka "0.6.0" + :exclusions [org.apache.zookeeper/zookeeper]]] :dev-dependencies [[storm "0.7.0-SNAPSHOT"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) From 331d20229de450bc0930bb5ac081c4f8d638e34d Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 18 Jan 2012 13:55:30 -0800 Subject: [PATCH 035/349] add readme for storm-kafka --- README.markdown | 1 + 1 file changed, 1 insertion(+) create mode 100644 README.markdown diff --git a/README.markdown b/README.markdown new file mode 100644 index 0000000000..a04a31d3b9 --- /dev/null +++ b/README.markdown @@ -0,0 +1 @@ +storm-kafka provides a TransactionalSpout implementation for Apache Kafka. This is currently unfinished, and TransactionalSpout won't be available in Storm until 0.7.0. \ No newline at end of file From 4e2dd49ee0c7bdfc78863d648e51d7c99b68f48c Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 20 Jan 2012 00:15:30 -0800 Subject: [PATCH 036/349] upgrade to 0.6.2 --- m2-pom.xml | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/m2-pom.xml b/m2-pom.xml index c2039e03ca..351a98dac9 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -38,7 +38,7 @@ storm storm - 0.6.0 + 0.6.2 provided diff --git a/project.clj b/project.clj index de52d5fc3c..31a66ad18a 100644 --- a/project.clj +++ b/project.clj @@ -15,6 +15,6 @@ [org.twitter4j/twitter4j-stream "2.2.5-SNAPSHOT"] ] - :dev-dependencies [[storm "0.6.0"] + :dev-dependencies [[storm "0.6.2"] ]) From d6e352a2c708b63e5ef65032092ac61713fee7cf Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 26 Jan 2012 20:35:49 -0800 Subject: [PATCH 037/349] update to new api --- src/jvm/storm/kafka/TransactionalKafkaSpout.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/jvm/storm/kafka/TransactionalKafkaSpout.java b/src/jvm/storm/kafka/TransactionalKafkaSpout.java index 4d9b636e6a..df20f774ce 100644 --- a/src/jvm/storm/kafka/TransactionalKafkaSpout.java +++ b/src/jvm/storm/kafka/TransactionalKafkaSpout.java @@ -6,8 +6,8 @@ import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BasePartitionedTransactionalSpout; import backtype.storm.transactional.TransactionAttempt; -import backtype.storm.transactional.TransactionalOutputCollector; import backtype.storm.transactional.partitioned.IPartitionedTransactionalSpout; +import backtype.storm.coordination.BatchOutputCollector; import backtype.storm.tuple.Fields; import backtype.storm.utils.Utils; import java.io.Serializable; @@ -63,7 +63,7 @@ class Emitter implements IPartitionedTransactionalSpout.Emitter { Map _kafka = new HashMap(); @Override - public BatchMeta emitPartitionBatchNew(TransactionAttempt attempt, TransactionalOutputCollector collector, int partition, BatchMeta lastMeta) { + public BatchMeta emitPartitionBatchNew(TransactionAttempt attempt, BatchOutputCollector collector, int partition, BatchMeta lastMeta) { SimpleConsumer consumer = connect(partition); long offset = 0; @@ -84,7 +84,7 @@ public BatchMeta emitPartitionBatchNew(TransactionAttempt attempt, Transactional } @Override - public void emitPartitionBatch(TransactionAttempt attempt, TransactionalOutputCollector collector, int partition, BatchMeta meta) { + public void emitPartitionBatch(TransactionAttempt attempt, BatchOutputCollector collector, int partition, BatchMeta meta) { SimpleConsumer consumer = connect(partition); ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition % _config.partitionsPerHost, meta.offset, _config.fetchSizeBytes)); @@ -100,7 +100,7 @@ public void emitPartitionBatch(TransactionAttempt attempt, TransactionalOutputCo } } - private void emit(TransactionAttempt attempt, TransactionalOutputCollector collector, Message msg) { + private void emit(TransactionAttempt attempt, BatchOutputCollector collector, Message msg) { List values = _config.scheme.deserialize(Utils.toByteArray(msg.payload())); List toEmit = new ArrayList(); toEmit.add(attempt); From fa4792325957dbfc71ec31961b4ec9b75fa44909 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 26 Jan 2012 22:49:03 -0800 Subject: [PATCH 038/349] update notes on storm-kafka --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index a04a31d3b9..30e1e07539 100644 --- a/README.markdown +++ b/README.markdown @@ -1 +1 @@ -storm-kafka provides a TransactionalSpout implementation for Apache Kafka. This is currently unfinished, and TransactionalSpout won't be available in Storm until 0.7.0. \ No newline at end of file +storm-kafka provides a TransactionalSpout implementation for Apache Kafka. TransactionalSpout won't be available in Storm until 0.7.0. This implementation has only been lightly tested. \ No newline at end of file From 40595a3641e44904053e00a3056b379f738a6100 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 27 Jan 2012 10:49:43 -0800 Subject: [PATCH 039/349] exclude log4j from dependencies since that's included through storm dependency --- project.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 716b9f7b65..b2897fa240 100644 --- a/project.clj +++ b/project.clj @@ -3,7 +3,8 @@ :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} :dependencies [[storm/kafka "0.6.0" - :exclusions [org.apache.zookeeper/zookeeper]]] + :exclusions [org.apache.zookeeper/zookeeper + log4j/log4j]]] :dev-dependencies [[storm "0.7.0-SNAPSHOT"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) From 8fc8c7c80c84cefdc428f45ca04bbc50cd309704 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 28 Jan 2012 02:32:18 -0800 Subject: [PATCH 040/349] updated to storm 0.7.0 and added example transactional topologies --- .../starter/TransactionalGlobalCount.java | 169 +++++++++++++++ .../storm/starter/TransactionalTopWords.java | 202 ++++++++++++++++++ 2 files changed, 371 insertions(+) create mode 100644 src/jvm/storm/starter/TransactionalGlobalCount.java create mode 100644 src/jvm/storm/starter/TransactionalTopWords.java diff --git a/src/jvm/storm/starter/TransactionalGlobalCount.java b/src/jvm/storm/starter/TransactionalGlobalCount.java new file mode 100644 index 0000000000..02596612a1 --- /dev/null +++ b/src/jvm/storm/starter/TransactionalGlobalCount.java @@ -0,0 +1,169 @@ +package storm.starter; + +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.coordination.BatchOutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.testing.MemoryTransactionalSpout; +import backtype.storm.topology.BasicOutputCollector; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseBasicBolt; +import backtype.storm.topology.base.BaseBatchBolt; +import backtype.storm.transactional.TransactionAttempt; +import backtype.storm.transactional.TransactionalTopologyBuilder; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class TransactionalGlobalCount { + public static final int PARTITION_TAKE_PER_BATCH = 3; + public static final Map>> DATA = new HashMap>>() {{ + put(0, new ArrayList>() {{ + add(new Values("cat")); + add(new Values("dog")); + add(new Values("chicken")); + add(new Values("cat")); + add(new Values("dog")); + add(new Values("apple")); + }}); + put(1, new ArrayList>() {{ + add(new Values("cat")); + add(new Values("dog")); + add(new Values("apple")); + add(new Values("banana")); + }}); + put(2, new ArrayList>() {{ + add(new Values("cat")); + add(new Values("cat")); + add(new Values("cat")); + add(new Values("cat")); + add(new Values("cat")); + add(new Values("dog")); + add(new Values("dog")); + add(new Values("dog")); + add(new Values("dog")); + }}); + }}; + + public static class Value { + int count = 0; + BigInteger txid; + } + + public static Map DATABASE = new HashMap(); + public static final String GLOBAL_COUNT_KEY = "GLOBAL-COUNT"; + + public static class BatchCount extends BaseBatchBolt { + Object _id; + BatchOutputCollector _collector; + + int _count = 0; + + @Override + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { + _collector = collector; + _id = id; + } + + @Override + public void execute(Tuple tuple) { + _count++; + } + + @Override + public void finishBatch() { + _collector.emit(new Values(_id, _count)); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "count")); + } + } + + public static class BatchSum extends BaseBatchBolt { + Object _id; + BatchOutputCollector _collector; + + int _sum = 0; + + @Override + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { + _collector = collector; + _id = id; + } + + @Override + public void execute(Tuple tuple) { + _sum+=tuple.getInteger(1); + } + + @Override + public void finishBatch() { + _collector.emit(new Values(_id, _sum)); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "sum")); + } + } + + public static class CommitToDB extends BaseBasicBolt { + + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + BigInteger txid = ((TransactionAttempt) tuple.getValue(0)).getTransactionId(); + int sum = tuple.getInteger(1); + + Value val = DATABASE.get(GLOBAL_COUNT_KEY); + Value newval; + if(val == null || !val.txid.equals(txid)) { + newval = new Value(); + newval.txid = txid; + if(val==null) { + newval.count = sum; + } else { + newval.count = sum + val.count; + } + DATABASE.put(GLOBAL_COUNT_KEY, newval); + } else { + newval = val; + } + collector.emit(new Values(tuple.getValue(0), newval.count)); + + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("attempt", "global-count")); + } + } + + public static void main(String[] args) throws Exception { + MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH); + TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 2); + builder.setBolt("partial-count", new BatchCount(), 5) + .shuffleGrouping("spout"); + builder.setCommitterBolt("sum", new BatchSum()) + .globalGrouping("partial-count"); + builder.setBolt("to-db", new CommitToDB()) + .shuffleGrouping("sum"); + + LocalCluster cluster = new LocalCluster(); + + Config config = new Config(); + config.setDebug(true); + config.setMaxSpoutPending(3); + + cluster.submitTopology("global-count-topology", config, builder.buildTopology()); + + Thread.sleep(3000); + cluster.shutdown(); + } +} diff --git a/src/jvm/storm/starter/TransactionalTopWords.java b/src/jvm/storm/starter/TransactionalTopWords.java new file mode 100644 index 0000000000..3ddd575a33 --- /dev/null +++ b/src/jvm/storm/starter/TransactionalTopWords.java @@ -0,0 +1,202 @@ +package storm.starter; + +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.coordination.BatchOutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.testing.MemoryTransactionalSpout; +import backtype.storm.topology.BasicOutputCollector; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseBasicBolt; +import backtype.storm.topology.base.BaseBatchBolt; +import backtype.storm.topology.base.BaseTransactionalBolt; +import backtype.storm.transactional.ICommitter; +import backtype.storm.transactional.TransactionAttempt; +import backtype.storm.transactional.TransactionalTopologyBuilder; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class TransactionalTopWords { + public static final int PARTITION_TAKE_PER_BATCH = 3; + public static final int TOP_AMOUNT = 10; + + public static final Map>> DATA = new HashMap>>() {{ + put(0, new ArrayList>() {{ + add(new Values("cat")); + add(new Values("dog")); + add(new Values("chicken")); + add(new Values("cat")); + add(new Values("dog")); + add(new Values("apple")); + }}); + put(1, new ArrayList>() {{ + add(new Values("cat")); + add(new Values("dog")); + add(new Values("apple")); + add(new Values("banana")); + }}); + put(2, new ArrayList>() {{ + add(new Values("cat")); + add(new Values("cat")); + add(new Values("cat")); + add(new Values("cat")); + add(new Values("cat")); + add(new Values("dog")); + add(new Values("dog")); + add(new Values("dog")); + add(new Values("dog")); + }}); + }}; + + public static class Value { + int count = 0; + BigInteger txid; + } + + public static Map DATABASE = new HashMap(); + public static String TOP_KEY = "__TOP"; + + public static class KeyedCountUpdater extends BaseTransactionalBolt implements ICommitter { + Map _counts = new HashMap(); + BatchOutputCollector _collector; + TransactionAttempt _id; + + int _count = 0; + + @Override + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, TransactionAttempt id) { + _collector = collector; + _id = id; + } + + @Override + public void execute(Tuple tuple) { + String key = tuple.getString(0); + Integer curr = _counts.get(key); + if(curr==null) curr = 0; + _counts.put(key, curr + 1); + } + + @Override + public void finishBatch() { + for(String key: _counts.keySet()) { + Value val = DATABASE.get(key); + if(!val.txid.equals(_id)) { + val.txid = _id.getTransactionId(); + val.count = val.count + _counts.get(key); + DATABASE.put(key, val); + } + _collector.emit(new Values(_id, key, val.count)); + } + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "key", "count")); + } + } + + public static class RankN extends BaseBatchBolt { + BatchOutputCollector _collector; + Object _id; + int _n; + + public RankN(int n) { + _n = n; + } + + @Override + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { + _collector = collector; + _id = id; + } + + @Override + public void execute(Tuple tuple) { + // TODO: finish + } + + @Override + public void finishBatch() { + // TODO: finish + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "list")); + } + } + + public static class MergeN extends BaseBatchBolt { + BatchOutputCollector _collector; + Object _id; + int _n; + + public MergeN(int n) { + _n = n; + } + + @Override + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { + _collector = collector; + _id = id; + } + + @Override + public void execute(Tuple tuple) { + // TODO: finish + } + + @Override + public void finishBatch() { + // TODO: finish + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "list")); + } + } + + public static class WriteTopN extends BaseBasicBolt { + + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + // TODO: get and merge with the list from the database + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + } + } + + public static void main(String[] args) throws Exception { + MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH); + TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("top-n-words", "spout", spout, 2); + builder.setBolt("count", new KeyedCountUpdater(), 5) + .fieldsGrouping("spout", new Fields("word")); + builder.setBolt("rank", new RankN(TOP_AMOUNT), 5) + .shuffleGrouping("count"); + builder.setBolt("merge", new MergeN(TOP_AMOUNT)) + .globalGrouping("rank"); + builder.setBolt("write-top-n", new WriteTopN()) + .shuffleGrouping("merge"); + + LocalCluster cluster = new LocalCluster(); + + Config config = new Config(); + config.setDebug(true); + config.setMaxSpoutPending(3); + + cluster.submitTopology("top-n-topology", config, builder.buildTopology()); + + Thread.sleep(3000); + cluster.shutdown(); + } +} From b8d043b00723e077501f5615be096226dcdf8c2c Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 28 Jan 2012 15:20:59 -0800 Subject: [PATCH 041/349] update to 0.7.0 and provide examples of transactional topologies --- .gitignore | 2 + project.clj | 2 +- src/jvm/storm/starter/BasicDRPCTopology.java | 11 +- .../storm/starter/ExclamationTopology.java | 8 +- src/jvm/storm/starter/ReachTopology.java | 93 +++------- ...lTopWords.java => TransactionalWords.java} | 160 ++++++++++-------- src/jvm/storm/starter/WordCountTopology.java | 18 +- src/jvm/storm/starter/bolt/MergeObjects.java | 12 +- src/jvm/storm/starter/bolt/PrinterBolt.java | 14 +- src/jvm/storm/starter/bolt/RankObjects.java | 10 +- .../starter/bolt/RollingCountObjects.java | 7 +- .../storm/starter/bolt/SingleJoinBolt.java | 8 +- .../starter/spout/RandomSentenceSpout.java | 16 +- .../starter/spout/TwitterSampleSpout.java | 17 +- 14 files changed, 151 insertions(+), 227 deletions(-) rename src/jvm/storm/starter/{TransactionalTopWords.java => TransactionalWords.java} (54%) diff --git a/.gitignore b/.gitignore index 6206669be2..c4b88f3834 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ lib/ .classpath .project *.jar +.lein-deps-sum + diff --git a/project.clj b/project.clj index 31a66ad18a..f9a8b511ea 100644 --- a/project.clj +++ b/project.clj @@ -15,6 +15,6 @@ [org.twitter4j/twitter4j-stream "2.2.5-SNAPSHOT"] ] - :dev-dependencies [[storm "0.6.2"] + :dev-dependencies [[storm "0.7.0-SNAPSHOT"] ]) diff --git a/src/jvm/storm/starter/BasicDRPCTopology.java b/src/jvm/storm/starter/BasicDRPCTopology.java index 718add223e..1ea8e7760b 100644 --- a/src/jvm/storm/starter/BasicDRPCTopology.java +++ b/src/jvm/storm/starter/BasicDRPCTopology.java @@ -7,8 +7,8 @@ import backtype.storm.drpc.LinearDRPCTopologyBuilder; import backtype.storm.task.TopologyContext; import backtype.storm.topology.BasicOutputCollector; -import backtype.storm.topology.IBasicBolt; import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseBasicBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; @@ -22,20 +22,13 @@ * doing distributed RPC on top of Storm. */ public class BasicDRPCTopology { - public static class ExclaimBolt implements IBasicBolt { - public void prepare(Map conf, TopologyContext context) { - } - + public static class ExclaimBolt extends BaseBasicBolt { @Override public void execute(Tuple tuple, BasicOutputCollector collector) { String input = tuple.getString(1); collector.emit(new Values(tuple.getValue(0), input + "!")); } - @Override - public void cleanup() { - } - @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("id", "result")); diff --git a/src/jvm/storm/starter/ExclamationTopology.java b/src/jvm/storm/starter/ExclamationTopology.java index 8a5b381c24..b98c8cd284 100644 --- a/src/jvm/storm/starter/ExclamationTopology.java +++ b/src/jvm/storm/starter/ExclamationTopology.java @@ -6,9 +6,9 @@ import backtype.storm.task.OutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.testing.TestWordSpout; -import backtype.storm.topology.IRichBolt; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.TopologyBuilder; +import backtype.storm.topology.base.BaseRichBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; @@ -20,7 +20,7 @@ */ public class ExclamationTopology { - public static class ExclamationBolt implements IRichBolt { + public static class ExclamationBolt extends BaseRichBolt { OutputCollector _collector; @Override @@ -34,10 +34,6 @@ public void execute(Tuple tuple) { _collector.ack(tuple); } - @Override - public void cleanup() { - } - @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("word")); diff --git a/src/jvm/storm/starter/ReachTopology.java b/src/jvm/storm/starter/ReachTopology.java index 82aa772379..8a7fdbcefc 100644 --- a/src/jvm/storm/starter/ReachTopology.java +++ b/src/jvm/storm/starter/ReachTopology.java @@ -4,14 +4,16 @@ import backtype.storm.LocalCluster; import backtype.storm.LocalDRPC; import backtype.storm.StormSubmitter; -import backtype.storm.drpc.CoordinatedBolt.FinishedCallback; +import backtype.storm.coordination.BatchOutputCollector; +import backtype.storm.coordination.CoordinatedBolt.FinishedCallback; import backtype.storm.drpc.LinearDRPCTopologyBuilder; import backtype.storm.task.OutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.topology.BasicOutputCollector; -import backtype.storm.topology.IBasicBolt; import backtype.storm.topology.IRichBolt; import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseBasicBolt; +import backtype.storm.topology.base.BaseBatchBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; @@ -57,11 +59,7 @@ public class ReachTopology { put("john", Arrays.asList("alice", "nathan", "jim", "mike", "bob")); }}; - public static class GetTweeters implements IBasicBolt { - @Override - public void prepare(Map conf, TopologyContext context) { - } - + public static class GetTweeters extends BaseBasicBolt { @Override public void execute(Tuple tuple, BasicOutputCollector collector) { Object id = tuple.getValue(0); @@ -74,21 +72,13 @@ public void execute(Tuple tuple, BasicOutputCollector collector) { } } - @Override - public void cleanup() { - } - @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("id", "tweeter")); } } - public static class GetFollowers implements IBasicBolt { - @Override - public void prepare(Map conf, TopologyContext context) { - } - + public static class GetFollowers extends BaseBasicBolt { @Override public void execute(Tuple tuple, BasicOutputCollector collector) { Object id = tuple.getValue(0); @@ -101,51 +91,31 @@ public void execute(Tuple tuple, BasicOutputCollector collector) { } } - @Override - public void cleanup() { - } - @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("id", "follower")); } } - public static class PartialUniquer implements IRichBolt, FinishedCallback { - OutputCollector _collector; - Map> _sets = new HashMap>(); + public static class PartialUniquer extends BaseBatchBolt { + BatchOutputCollector _collector; + Object _id; + Set _followers = new HashSet(); @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { _collector = collector; + _id = id; } @Override public void execute(Tuple tuple) { - Object id = tuple.getValue(0); - Set curr = _sets.get(id); - if(curr==null) { - curr = new HashSet(); - _sets.put(id, curr); - } - curr.add(tuple.getString(1)); - _collector.ack(tuple); + _followers.add(tuple.getString(1)); } - - @Override - public void cleanup() { - } - + @Override - public void finishedId(Object id) { - Set curr = _sets.remove(id); - int count; - if(curr!=null) { - count = curr.size(); - } else { - count = 0; - } - _collector.emit(new Values(id, count)); + public void finishBatch() { + _collector.emit(new Values(_id, _followers.size())); } @Override @@ -154,42 +124,31 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } } - public static class CountAggregator implements IRichBolt, FinishedCallback { - Map _counts = new HashMap(); - OutputCollector _collector; + public static class CountAggregator extends BaseBatchBolt { + BatchOutputCollector _collector; + Object _id; + int _count = 0; @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { _collector = collector; + _id = id; } @Override public void execute(Tuple tuple) { - Object id = tuple.getValue(0); - int partial = tuple.getInteger(1); - - Integer curr = _counts.get(id); - if(curr==null) curr = 0; - _counts.put(id, curr + partial); - _collector.ack(tuple); - } - - @Override - public void cleanup() { + _count+=tuple.getInteger(1); } - + @Override - public void finishedId(Object id) { - Integer reach = _counts.get(id); - if(reach==null) reach = 0; - _collector.emit(new Values(id, reach)); + public void finishBatch() { + _collector.emit(new Values(_id, _count)); } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("id", "reach")); } - } public static LinearDRPCTopologyBuilder construct() { diff --git a/src/jvm/storm/starter/TransactionalTopWords.java b/src/jvm/storm/starter/TransactionalWords.java similarity index 54% rename from src/jvm/storm/starter/TransactionalTopWords.java rename to src/jvm/storm/starter/TransactionalWords.java index 3ddd575a33..5dd4fe39d6 100644 --- a/src/jvm/storm/starter/TransactionalTopWords.java +++ b/src/jvm/storm/starter/TransactionalWords.java @@ -8,7 +8,6 @@ import backtype.storm.topology.BasicOutputCollector; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseBasicBolt; -import backtype.storm.topology.base.BaseBatchBolt; import backtype.storm.topology.base.BaseTransactionalBolt; import backtype.storm.transactional.ICommitter; import backtype.storm.transactional.TransactionAttempt; @@ -22,9 +21,26 @@ import java.util.List; import java.util.Map; -public class TransactionalTopWords { + +public class TransactionalWords { + public static class CountValue { + Integer prev_count = null; + int count = 0; + BigInteger txid = null; + } + + public static class BucketValue { + int count = 0; + BigInteger txid; + } + + public static final int BUCKET_SIZE = 10; + + public static Map COUNT_DATABASE = new HashMap(); + public static Map BUCKET_DATABASE = new HashMap(); + + public static final int PARTITION_TAKE_PER_BATCH = 3; - public static final int TOP_AMOUNT = 10; public static final Map>> DATA = new HashMap>>() {{ put(0, new ArrayList>() {{ @@ -53,15 +69,7 @@ public class TransactionalTopWords { add(new Values("dog")); }}); }}; - - public static class Value { - int count = 0; - BigInteger txid; - } - - public static Map DATABASE = new HashMap(); - public static String TOP_KEY = "__TOP"; - + public static class KeyedCountUpdater extends BaseTransactionalBolt implements ICommitter { Map _counts = new HashMap(); BatchOutputCollector _collector; @@ -77,7 +85,7 @@ public void prepare(Map conf, TopologyContext context, BatchOutputCollector coll @Override public void execute(Tuple tuple) { - String key = tuple.getString(0); + String key = tuple.getString(1); Integer curr = _counts.get(key); if(curr==null) curr = 0; _counts.put(key, curr + 1); @@ -86,93 +94,100 @@ public void execute(Tuple tuple) { @Override public void finishBatch() { for(String key: _counts.keySet()) { - Value val = DATABASE.get(key); - if(!val.txid.equals(_id)) { - val.txid = _id.getTransactionId(); - val.count = val.count + _counts.get(key); - DATABASE.put(key, val); + CountValue val = COUNT_DATABASE.get(key); + CountValue newVal; + if(val==null || !val.txid.equals(_id)) { + newVal = new CountValue(); + newVal.txid = _id.getTransactionId(); + if(val!=null) { + newVal.prev_count = val.count; + newVal.count = val.count; + } + newVal.count = newVal.count + _counts.get(key); + COUNT_DATABASE.put(key, val); + } else { + newVal = val; } - _collector.emit(new Values(_id, key, val.count)); + _collector.emit(new Values(_id, key, newVal.count, newVal.prev_count)); } } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("id", "key", "count")); + declarer.declare(new Fields("id", "key", "count", "prev-count")); } } - public static class RankN extends BaseBatchBolt { - BatchOutputCollector _collector; - Object _id; - int _n; - - public RankN(int n) { - _n = n; - } - + public static class Bucketize extends BaseBasicBolt { @Override - public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { - _collector = collector; - _id = id; - } - - @Override - public void execute(Tuple tuple) { - // TODO: finish - } + public void execute(Tuple tuple, BasicOutputCollector collector) { + TransactionAttempt attempt = (TransactionAttempt) tuple.getValue(0); + int curr = tuple.getInteger(2); + Integer prev = tuple.getInteger(3); - @Override - public void finishBatch() { - // TODO: finish + int currBucket = curr / BUCKET_SIZE; + Integer prevBucket = null; + if(prev!=null) { + prevBucket = prev / BUCKET_SIZE; + } + + if(prevBucket==null) { + collector.emit(new Values(attempt, currBucket, 1)); + } else if(currBucket != prevBucket) { + collector.emit(new Values(attempt, currBucket, 1)); + collector.emit(new Values(attempt, prevBucket, -1)); + } } - + @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("id", "list")); + declarer.declare(new Fields("attempt", "bucket", "delta")); } } - public static class MergeN extends BaseBatchBolt { + public static class BucketCountUpdater extends BaseTransactionalBolt { + Map _accum = new HashMap(); BatchOutputCollector _collector; - Object _id; - int _n; - - public MergeN(int n) { - _n = n; - } + TransactionAttempt _attempt; + int _count = 0; + @Override - public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, TransactionAttempt attempt) { _collector = collector; - _id = id; + _attempt = attempt; } @Override public void execute(Tuple tuple) { - // TODO: finish + Integer bucket = tuple.getInteger(1); + Integer delta = tuple.getInteger(2); + Integer curr = _accum.get(bucket); + if(curr==null) curr = 0; + _accum.put(bucket, curr + delta); } @Override public void finishBatch() { - // TODO: finish - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("id", "list")); - } - } - - public static class WriteTopN extends BaseBasicBolt { - - @Override - public void execute(Tuple tuple, BasicOutputCollector collector) { - // TODO: get and merge with the list from the database + for(Integer bucket: _accum.keySet()) { + BucketValue currVal = BUCKET_DATABASE.get(bucket); + BucketValue newVal; + if(currVal==null || !currVal.txid.equals(_attempt.getTransactionId())) { + newVal = new BucketValue(); + newVal.txid = _attempt.getTransactionId(); + newVal.count = _accum.get(bucket); + if(currVal!=null) newVal.count += currVal.count; + BUCKET_DATABASE.put(bucket, newVal); + } else { + newVal = currVal; + } + _collector.emit(new Values(_attempt, bucket, newVal.count)); + } } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "bucket", "count")); } } @@ -181,12 +196,11 @@ public static void main(String[] args) throws Exception { TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("top-n-words", "spout", spout, 2); builder.setBolt("count", new KeyedCountUpdater(), 5) .fieldsGrouping("spout", new Fields("word")); - builder.setBolt("rank", new RankN(TOP_AMOUNT), 5) - .shuffleGrouping("count"); - builder.setBolt("merge", new MergeN(TOP_AMOUNT)) - .globalGrouping("rank"); - builder.setBolt("write-top-n", new WriteTopN()) - .shuffleGrouping("merge"); + builder.setBolt("bucketize", new Bucketize()) + .noneGrouping("count"); + builder.setBolt("buckets", new BucketCountUpdater(), 5) + .fieldsGrouping("bucketize", new Fields("bucket")); + LocalCluster cluster = new LocalCluster(); diff --git a/src/jvm/storm/starter/WordCountTopology.java b/src/jvm/storm/starter/WordCountTopology.java index 9ed988de0d..bed968d908 100644 --- a/src/jvm/storm/starter/WordCountTopology.java +++ b/src/jvm/storm/starter/WordCountTopology.java @@ -7,10 +7,10 @@ import backtype.storm.task.ShellBolt; import backtype.storm.task.TopologyContext; import backtype.storm.topology.BasicOutputCollector; -import backtype.storm.topology.IBasicBolt; import backtype.storm.topology.IRichBolt; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.TopologyBuilder; +import backtype.storm.topology.base.BaseBasicBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; @@ -31,14 +31,15 @@ public SplitSentence() { public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("word")); } - } - - public static class WordCount implements IBasicBolt { - Map counts = new HashMap(); @Override - public void prepare(Map conf, TopologyContext context) { + public Map getComponentConfiguration() { + return null; } + } + + public static class WordCount extends BaseBasicBolt { + Map counts = new HashMap(); @Override public void execute(Tuple tuple, BasicOutputCollector collector) { @@ -50,10 +51,6 @@ public void execute(Tuple tuple, BasicOutputCollector collector) { collector.emit(new Values(word, count)); } - @Override - public void cleanup() { - } - @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("word", "count")); @@ -84,6 +81,7 @@ public static void main(String[] args) throws Exception { LocalCluster cluster = new LocalCluster(); cluster.submitTopology("word-count", conf, builder.createTopology()); + Thread.sleep(10000); cluster.shutdown(); diff --git a/src/jvm/storm/starter/bolt/MergeObjects.java b/src/jvm/storm/starter/bolt/MergeObjects.java index 97bdaf8f77..a536aeb887 100644 --- a/src/jvm/storm/starter/bolt/MergeObjects.java +++ b/src/jvm/storm/starter/bolt/MergeObjects.java @@ -3,8 +3,8 @@ import org.apache.log4j.Logger; import backtype.storm.task.TopologyContext; import backtype.storm.topology.BasicOutputCollector; -import backtype.storm.topology.IBasicBolt; import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseBasicBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; @@ -14,7 +14,7 @@ import java.util.List; import java.util.Map; -public class MergeObjects implements IBasicBolt { +public class MergeObjects extends BaseBasicBolt { public static Logger LOG = Logger.getLogger(MergeObjects.class); private List _rankings = new ArrayList(); @@ -48,11 +48,6 @@ private Integer _find(Object tag) { return null; } - - public void prepare(Map stormConf, TopologyContext context) { - - } - public void execute(Tuple tuple, BasicOutputCollector collector) { List merging = (List) tuple.getValue(0); for(List pair : merging) { @@ -82,9 +77,6 @@ public int compare(List o1, List o2) { } } - public void cleanup() { - } - public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("list")); } diff --git a/src/jvm/storm/starter/bolt/PrinterBolt.java b/src/jvm/storm/starter/bolt/PrinterBolt.java index 66d6262f57..ac380ed6a2 100644 --- a/src/jvm/storm/starter/bolt/PrinterBolt.java +++ b/src/jvm/storm/starter/bolt/PrinterBolt.java @@ -1,28 +1,18 @@ package storm.starter.bolt; -import backtype.storm.task.TopologyContext; import backtype.storm.topology.BasicOutputCollector; -import backtype.storm.topology.IBasicBolt; import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseBasicBolt; import backtype.storm.tuple.Tuple; -import java.util.Map; -public class PrinterBolt implements IBasicBolt { - - @Override - public void prepare(Map conf, TopologyContext context) { - } +public class PrinterBolt extends BaseBasicBolt { @Override public void execute(Tuple tuple, BasicOutputCollector collector) { System.out.println(tuple); } - @Override - public void cleanup() { - } - @Override public void declareOutputFields(OutputFieldsDeclarer ofd) { } diff --git a/src/jvm/storm/starter/bolt/RankObjects.java b/src/jvm/storm/starter/bolt/RankObjects.java index e6c9e501f0..95455db9b8 100644 --- a/src/jvm/storm/starter/bolt/RankObjects.java +++ b/src/jvm/storm/starter/bolt/RankObjects.java @@ -2,8 +2,8 @@ import backtype.storm.task.TopologyContext; import backtype.storm.topology.BasicOutputCollector; -import backtype.storm.topology.IBasicBolt; import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseBasicBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; @@ -14,7 +14,7 @@ import java.util.Map; -public class RankObjects implements IBasicBolt { +public class RankObjects extends BaseBasicBolt { List _rankings = new ArrayList(); int _count; @@ -47,9 +47,6 @@ private Integer _find(Object tag) { return null; } - public void prepare(Map stormConf, TopologyContext context) { - } - public void execute(Tuple tuple, BasicOutputCollector collector) { Object tag = tuple.getValue(0); Integer existingIndex = _find(tag); @@ -73,9 +70,6 @@ public int compare(List o1, List o2) { } } - public void cleanup() { - } - public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("list")); } diff --git a/src/jvm/storm/starter/bolt/RollingCountObjects.java b/src/jvm/storm/starter/bolt/RollingCountObjects.java index 55d73e013a..a0c06f31e0 100644 --- a/src/jvm/storm/starter/bolt/RollingCountObjects.java +++ b/src/jvm/storm/starter/bolt/RollingCountObjects.java @@ -2,8 +2,8 @@ import backtype.storm.task.OutputCollector; import backtype.storm.task.TopologyContext; -import backtype.storm.topology.IRichBolt; import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseRichBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; @@ -13,7 +13,7 @@ import java.util.Map; import java.util.Set; -public class RollingCountObjects implements IRichBolt { +public class RollingCountObjects extends BaseRichBolt { private HashMap _objectCounts = new HashMap(); private int _numBuckets; @@ -103,9 +103,6 @@ public void execute(Tuple tuple) { } } - public void cleanup() { - } - public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("obj", "count")); } diff --git a/src/jvm/storm/starter/bolt/SingleJoinBolt.java b/src/jvm/storm/starter/bolt/SingleJoinBolt.java index df944e0737..e83b9be196 100644 --- a/src/jvm/storm/starter/bolt/SingleJoinBolt.java +++ b/src/jvm/storm/starter/bolt/SingleJoinBolt.java @@ -4,8 +4,8 @@ import backtype.storm.generated.GlobalStreamId; import backtype.storm.task.OutputCollector; import backtype.storm.task.TopologyContext; -import backtype.storm.topology.IRichBolt; import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseRichBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.utils.TimeCacheMap; @@ -16,7 +16,7 @@ import java.util.Map; import java.util.Set; -public class SingleJoinBolt implements IRichBolt { +public class SingleJoinBolt extends BaseRichBolt { OutputCollector _collector; Fields _idFields; Fields _outFields; @@ -82,10 +82,6 @@ public void execute(Tuple tuple) { } } - @Override - public void cleanup() { - } - @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(_outFields); diff --git a/src/jvm/storm/starter/spout/RandomSentenceSpout.java b/src/jvm/storm/starter/spout/RandomSentenceSpout.java index 1a48db1a99..af6c151d8d 100644 --- a/src/jvm/storm/starter/spout/RandomSentenceSpout.java +++ b/src/jvm/storm/starter/spout/RandomSentenceSpout.java @@ -2,22 +2,18 @@ import backtype.storm.spout.SpoutOutputCollector; import backtype.storm.task.TopologyContext; -import backtype.storm.topology.IRichSpout; import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseRichSpout; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Values; import backtype.storm.utils.Utils; import java.util.Map; import java.util.Random; -public class RandomSentenceSpout implements IRichSpout { +public class RandomSentenceSpout extends BaseRichSpout { SpoutOutputCollector _collector; Random _rand; - @Override - public boolean isDistributed() { - return true; - } @Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { @@ -36,13 +32,7 @@ public void nextTuple() { "i am at two with nature"}; String sentence = sentences[_rand.nextInt(sentences.length)]; _collector.emit(new Values(sentence)); - } - - - @Override - public void close() { - } - + } @Override public void ack(Object id) { diff --git a/src/jvm/storm/starter/spout/TwitterSampleSpout.java b/src/jvm/storm/starter/spout/TwitterSampleSpout.java index 172b1a47d5..4c0e1332b0 100644 --- a/src/jvm/storm/starter/spout/TwitterSampleSpout.java +++ b/src/jvm/storm/starter/spout/TwitterSampleSpout.java @@ -1,12 +1,13 @@ package storm.starter.spout; +import backtype.storm.Config; import twitter4j.conf.ConfigurationBuilder; import twitter4j.TwitterStream; import twitter4j.TwitterStreamFactory; import backtype.storm.spout.SpoutOutputCollector; import backtype.storm.task.TopologyContext; -import backtype.storm.topology.IRichSpout; import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseRichSpout; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Values; import backtype.storm.utils.Utils; @@ -16,7 +17,7 @@ import twitter4j.StatusDeletionNotice; import twitter4j.StatusListener; -public class TwitterSampleSpout implements IRichSpout { +public class TwitterSampleSpout extends BaseRichSpout { SpoutOutputCollector _collector; LinkedBlockingQueue queue = null; TwitterStream _twitterStream; @@ -29,11 +30,6 @@ public TwitterSampleSpout(String username, String pwd) { _pwd = pwd; } - @Override - public boolean isDistributed() { - return false; - } - @Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { queue = new LinkedBlockingQueue(1000); @@ -83,6 +79,13 @@ public void close() { _twitterStream.shutdown(); } + @Override + public Map getComponentConfiguration() { + Config ret = new Config(); + ret.setMaxTaskParallelism(1); + return ret; + } + @Override public void ack(Object id) { } From b9fb42d581cc43170f90711f2e895f6d342ac7fd Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 28 Jan 2012 22:38:59 -0800 Subject: [PATCH 042/349] simplify example --- .../starter/TransactionalGlobalCount.java | 49 ++++++------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/src/jvm/storm/starter/TransactionalGlobalCount.java b/src/jvm/storm/starter/TransactionalGlobalCount.java index 02596612a1..c7643f2323 100644 --- a/src/jvm/storm/starter/TransactionalGlobalCount.java +++ b/src/jvm/storm/starter/TransactionalGlobalCount.java @@ -9,6 +9,8 @@ import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseBasicBolt; import backtype.storm.topology.base.BaseBatchBolt; +import backtype.storm.topology.base.BaseTransactionalBolt; +import backtype.storm.transactional.ICommitter; import backtype.storm.transactional.TransactionAttempt; import backtype.storm.transactional.TransactionalTopologyBuilder; import backtype.storm.tuple.Fields; @@ -86,16 +88,16 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } } - public static class BatchSum extends BaseBatchBolt { - Object _id; + public static class UpdateGlobalCount extends BaseTransactionalBolt implements ICommitter { + TransactionAttempt _attempt; BatchOutputCollector _collector; int _sum = 0; @Override - public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, TransactionAttempt attempt) { _collector = collector; - _id = id; + _attempt = attempt; } @Override @@ -105,55 +107,36 @@ public void execute(Tuple tuple) { @Override public void finishBatch() { - _collector.emit(new Values(_id, _sum)); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("id", "sum")); - } - } - - public static class CommitToDB extends BaseBasicBolt { - - @Override - public void execute(Tuple tuple, BasicOutputCollector collector) { - BigInteger txid = ((TransactionAttempt) tuple.getValue(0)).getTransactionId(); - int sum = tuple.getInteger(1); - Value val = DATABASE.get(GLOBAL_COUNT_KEY); Value newval; - if(val == null || !val.txid.equals(txid)) { + if(val == null || !val.txid.equals(_attempt.getTransactionId())) { newval = new Value(); - newval.txid = txid; + newval.txid = _attempt.getTransactionId(); if(val==null) { - newval.count = sum; + newval.count = _sum; } else { - newval.count = sum + val.count; + newval.count = _sum + val.count; } DATABASE.put(GLOBAL_COUNT_KEY, newval); } else { newval = val; } - collector.emit(new Values(tuple.getValue(0), newval.count)); - + _collector.emit(new Values(_attempt, newval.count)); } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("attempt", "global-count")); - } + declarer.declare(new Fields("id", "sum")); + } } public static void main(String[] args) throws Exception { MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH); - TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 2); + TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 3); builder.setBolt("partial-count", new BatchCount(), 5) - .shuffleGrouping("spout"); - builder.setCommitterBolt("sum", new BatchSum()) + .noneGrouping("spout"); + builder.setBolt("sum", new UpdateGlobalCount()) .globalGrouping("partial-count"); - builder.setBolt("to-db", new CommitToDB()) - .shuffleGrouping("sum"); LocalCluster cluster = new LocalCluster(); From 288d24a882bf4f87496eeeba01db5a0d4d569eea Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 28 Jan 2012 23:05:08 -0800 Subject: [PATCH 043/349] add comments to examples --- src/jvm/storm/starter/TransactionalGlobalCount.java | 7 +++++-- src/jvm/storm/starter/TransactionalWords.java | 12 +++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/jvm/storm/starter/TransactionalGlobalCount.java b/src/jvm/storm/starter/TransactionalGlobalCount.java index c7643f2323..b4579a4e13 100644 --- a/src/jvm/storm/starter/TransactionalGlobalCount.java +++ b/src/jvm/storm/starter/TransactionalGlobalCount.java @@ -5,9 +5,7 @@ import backtype.storm.coordination.BatchOutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.testing.MemoryTransactionalSpout; -import backtype.storm.topology.BasicOutputCollector; import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseBasicBolt; import backtype.storm.topology.base.BaseBatchBolt; import backtype.storm.topology.base.BaseTransactionalBolt; import backtype.storm.transactional.ICommitter; @@ -22,6 +20,11 @@ import java.util.List; import java.util.Map; +/** + * This is a basic example of a transactional topology. It keeps a count of the number of tuples seen so far + * in a database. The source of data and the databases are mocked out as in memory maps for demonstration + * purposes. This class is defined in depth on the wiki at https://github.com/nathanmarz/storm/wiki/Transactional-topologies + */ public class TransactionalGlobalCount { public static final int PARTITION_TAKE_PER_BATCH = 3; public static final Map>> DATA = new HashMap>>() {{ diff --git a/src/jvm/storm/starter/TransactionalWords.java b/src/jvm/storm/starter/TransactionalWords.java index 5dd4fe39d6..674600f660 100644 --- a/src/jvm/storm/starter/TransactionalWords.java +++ b/src/jvm/storm/starter/TransactionalWords.java @@ -21,7 +21,17 @@ import java.util.List; import java.util.Map; - +/** + * This class defines a more involved transactional topology then TransactionalGlobalCount. This topology + * processes a stream of words and produces two outputs: + * + * 1. A count for each word (stored in a database) + * 2. The number of words for every bucket of 10 counts. So it stores in the database how many words have appeared + * 0-9 times, how many have appeared 10-19 times, and so on. + * + * A batch of words can cause the bucket counts to decrement for some buckets and increment for others as words move + * between buckets as their counts accumulate. + */ public class TransactionalWords { public static class CountValue { Integer prev_count = null; From ec6c752274fe954652ae77a7f9645b3a7c8fc5df Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 28 Jan 2012 23:14:38 -0800 Subject: [PATCH 044/349] update README --- README.markdown | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 30e1e07539..59adb65ccd 100644 --- a/README.markdown +++ b/README.markdown @@ -1 +1,4 @@ -storm-kafka provides a TransactionalSpout implementation for Apache Kafka. TransactionalSpout won't be available in Storm until 0.7.0. This implementation has only been lightly tested. \ No newline at end of file +storm-kafka provides a TransactionalSpout implementation for Apache Kafka. TransactionalSpout won't be available in Storm until 0.7.0 (a development release is currently available though). This implementation has only been lightly tested. + +storm-kafka is available from Maven [here](http://clojars.org/storm/storm-kafka) + From c0078a3597ec43555fc846f31eb50fcf4514c1e8 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sun, 29 Jan 2012 02:30:42 -0800 Subject: [PATCH 045/349] note implementation is for 0.6 --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 59adb65ccd..b60cfff99e 100644 --- a/README.markdown +++ b/README.markdown @@ -1,4 +1,4 @@ -storm-kafka provides a TransactionalSpout implementation for Apache Kafka. TransactionalSpout won't be available in Storm until 0.7.0 (a development release is currently available though). This implementation has only been lightly tested. +storm-kafka provides a TransactionalSpout implementation for Apache Kafka 0.6. TransactionalSpout won't be available in Storm until 0.7.0 (a development release is currently available though). This implementation has only been lightly tested. storm-kafka is available from Maven [here](http://clojars.org/storm/storm-kafka) From ac2143ddb9dc9e68cf1ca46a45fbfc8234983adc Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 3 Feb 2012 00:39:19 -0800 Subject: [PATCH 046/349] kafka 7 compatible version --- project.clj | 4 ++-- .../storm/kafka/TransactionalKafkaSpout.java | 20 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/project.clj b/project.clj index b2897fa240..57644a39d8 100644 --- a/project.clj +++ b/project.clj @@ -1,8 +1,8 @@ -(defproject storm/storm-kafka "0.7.0-SNAPSHOT" +(defproject storm/storm-kafka "0.7.0-kafka7-SNAPSHOT" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} - :dependencies [[storm/kafka "0.6.0" + :dependencies [[storm/kafka "0.7.0-incubating" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] :dev-dependencies [[storm "0.7.0-SNAPSHOT"]] diff --git a/src/jvm/storm/kafka/TransactionalKafkaSpout.java b/src/jvm/storm/kafka/TransactionalKafkaSpout.java index df20f774ce..df0824c53f 100644 --- a/src/jvm/storm/kafka/TransactionalKafkaSpout.java +++ b/src/jvm/storm/kafka/TransactionalKafkaSpout.java @@ -19,7 +19,7 @@ import kafka.javaapi.consumer.SimpleConsumer; import kafka.javaapi.message.ByteBufferMessageSet; import kafka.message.Message; -import kafka.message.MessageSet; +import kafka.message.MessageAndOffset; public class TransactionalKafkaSpout extends BasePartitionedTransactionalSpout { @@ -73,9 +73,9 @@ public BatchMeta emitPartitionBatchNew(TransactionAttempt attempt, BatchOutputCo ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition % _config.partitionsPerHost, offset, _config.fetchSizeBytes)); long endoffset = offset; - for(Message msg: msgs) { - emit(attempt, collector, msg); - endoffset += MessageSet.entrySize(msg); + for(MessageAndOffset msg: msgs) { + emit(attempt, collector, msg.message()); + endoffset = msg.offset(); } BatchMeta newMeta = new BatchMeta(); newMeta.offset = offset; @@ -88,14 +88,14 @@ public void emitPartitionBatch(TransactionAttempt attempt, BatchOutputCollector SimpleConsumer consumer = connect(partition); ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition % _config.partitionsPerHost, meta.offset, _config.fetchSizeBytes)); - long currOffset = meta.offset; - for(Message msg: msgs) { - if(currOffset == meta.nextOffset) break; - if(currOffset > meta.nextOffset) { + long offset = meta.offset; + for(MessageAndOffset msg: msgs) { + if(offset == meta.nextOffset) break; + if(offset > meta.nextOffset) { throw new RuntimeException("Error when re-emitting batch. overshot the end offset"); } - emit(attempt, collector, msg); - currOffset += MessageSet.entrySize(msg); + emit(attempt, collector, msg.message()); + offset = msg.offset(); } } From 2708f8b74ee576e60131f10043dc19f2b65f45f3 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 3 Feb 2012 01:35:06 -0800 Subject: [PATCH 047/349] update README --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index b60cfff99e..3834a3cb60 100644 --- a/README.markdown +++ b/README.markdown @@ -1,4 +1,4 @@ -storm-kafka provides a TransactionalSpout implementation for Apache Kafka 0.6. TransactionalSpout won't be available in Storm until 0.7.0 (a development release is currently available though). This implementation has only been lightly tested. +storm-kafka provides a TransactionalSpout implementation for Apache Kafka 0.7. TransactionalSpout won't be available in Storm until 0.7.0 (a development release is currently available though). This implementation has only been lightly tested. storm-kafka is available from Maven [here](http://clojars.org/storm/storm-kafka) From 7b1eeb1b1a2b4199797416c52e425e5d893f49c7 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 3 Feb 2012 01:35:58 -0800 Subject: [PATCH 048/349] update README --- README.markdown | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index b60cfff99e..aceb1fbd62 100644 --- a/README.markdown +++ b/README.markdown @@ -1,4 +1,6 @@ -storm-kafka provides a TransactionalSpout implementation for Apache Kafka 0.6. TransactionalSpout won't be available in Storm until 0.7.0 (a development release is currently available though). This implementation has only been lightly tested. +storm-kafka provides a TransactionalSpout implementation for Apache Kafka. TransactionalSpout won't be available in Storm until 0.7.0 (a development release is currently available though). This implementation has only been lightly tested. + +The master branch is compatible with Kafka 0.6, the kafka0.7 branch is compatible with Kafka 0.7. storm-kafka is available from Maven [here](http://clojars.org/storm/storm-kafka) From 9b98f8a1c80632656b2afedeb62912fe35f61bc9 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 3 Feb 2012 03:14:32 -0800 Subject: [PATCH 049/349] instructions on getting getting 0.6 or 0.7 compatible version --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index aceb1fbd62..5f18b258d0 100644 --- a/README.markdown +++ b/README.markdown @@ -2,5 +2,5 @@ storm-kafka provides a TransactionalSpout implementation for Apache Kafka. Trans The master branch is compatible with Kafka 0.6, the kafka0.7 branch is compatible with Kafka 0.7. -storm-kafka is available from Maven [here](http://clojars.org/storm/storm-kafka) +storm-kafka is available from Maven [here](http://clojars.org/storm/storm-kafka). A Kafka 0.6 compatible version is under release "0.7-SNAPSHOT", and a Kafka 0.7 compatible version is under release "0.7.0-kafka7-SNAPSHOT". From 81074357185251ee61976b7f05aac4f6e61c2231 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 11 Feb 2012 17:44:04 -0800 Subject: [PATCH 050/349] added opaque version of spout for increased fault-tolerance --- project.clj | 2 +- src/jvm/storm/kafka/KafkaConfig.java | 23 ++++++ .../kafka/KafkaPartitionConnections.java | 30 ++++++++ src/jvm/storm/kafka/KafkaUtils.java | 40 +++++++++++ .../kafka/OpaqueTransactionalKafkaSpout.java | 66 +++++++++++++++++ src/jvm/storm/kafka/StringScheme.java | 22 ++++++ src/jvm/storm/kafka/TestTopology.java | 26 +++++++ .../storm/kafka/TransactionalKafkaSpout.java | 72 ++++--------------- 8 files changed, 220 insertions(+), 61 deletions(-) create mode 100644 src/jvm/storm/kafka/KafkaConfig.java create mode 100644 src/jvm/storm/kafka/KafkaPartitionConnections.java create mode 100644 src/jvm/storm/kafka/KafkaUtils.java create mode 100644 src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java create mode 100644 src/jvm/storm/kafka/StringScheme.java create mode 100644 src/jvm/storm/kafka/TestTopology.java diff --git a/project.clj b/project.clj index b2897fa240..314fbd4ada 100644 --- a/project.clj +++ b/project.clj @@ -5,6 +5,6 @@ :dependencies [[storm/kafka "0.6.0" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.7.0-SNAPSHOT"]] + :dev-dependencies [[storm "0.7.0-rc2-SNAPSHOT"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) diff --git a/src/jvm/storm/kafka/KafkaConfig.java b/src/jvm/storm/kafka/KafkaConfig.java new file mode 100644 index 0000000000..678d1ff0f3 --- /dev/null +++ b/src/jvm/storm/kafka/KafkaConfig.java @@ -0,0 +1,23 @@ +package storm.kafka; + +import backtype.storm.spout.RawScheme; +import backtype.storm.spout.Scheme; +import java.io.Serializable; +import java.util.List; + +public class KafkaConfig implements Serializable { + public List hosts; + public int port = 9092; + public int partitionsPerHost; + public int fetchSizeBytes = 1024*1024; + public int socketTimeoutMs = 10000; + public int bufferSizeBytes = 1024*1024; + public Scheme scheme = new RawScheme(); + public String topic; + + public KafkaConfig(List hosts, int partitionsPerHost, String topic) { + this.hosts = hosts; + this.partitionsPerHost = partitionsPerHost; + this.topic = topic; + } +} diff --git a/src/jvm/storm/kafka/KafkaPartitionConnections.java b/src/jvm/storm/kafka/KafkaPartitionConnections.java new file mode 100644 index 0000000000..25199f42b4 --- /dev/null +++ b/src/jvm/storm/kafka/KafkaPartitionConnections.java @@ -0,0 +1,30 @@ +package storm.kafka; + +import java.util.HashMap; +import java.util.Map; +import kafka.javaapi.consumer.SimpleConsumer; + +public class KafkaPartitionConnections { + Map _kafka = new HashMap(); + + KafkaConfig _config; + + public KafkaPartitionConnections(KafkaConfig conf) { + _config = conf; + } + + public SimpleConsumer getConsumer(int partition) { + if(!_kafka.containsKey(partition)) { + int hostIndex = partition % _config.hosts.size(); + _kafka.put(partition, new SimpleConsumer(_config.hosts.get(hostIndex), _config.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); + + } + return _kafka.get(partition); + } + + public void close() { + for(SimpleConsumer consumer: _kafka.values()) { + consumer.close(); + } + } +} diff --git a/src/jvm/storm/kafka/KafkaUtils.java b/src/jvm/storm/kafka/KafkaUtils.java new file mode 100644 index 0000000000..81d8f130ef --- /dev/null +++ b/src/jvm/storm/kafka/KafkaUtils.java @@ -0,0 +1,40 @@ +package storm.kafka; + +import backtype.storm.coordination.BatchOutputCollector; +import backtype.storm.transactional.TransactionAttempt; +import backtype.storm.utils.Utils; +import java.util.ArrayList; +import java.util.List; +import kafka.api.FetchRequest; +import kafka.javaapi.consumer.SimpleConsumer; +import kafka.javaapi.message.ByteBufferMessageSet; +import kafka.message.Message; +import kafka.message.MessageSet; + +public class KafkaUtils { + public static BatchMeta emitPartitionBatchNew(KafkaConfig config, int partition, SimpleConsumer consumer, TransactionAttempt attempt, BatchOutputCollector collector, BatchMeta lastMeta) { + long offset = 0; + if(lastMeta!=null) { + offset = lastMeta.nextOffset; + } + + ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(config.topic, partition % config.partitionsPerHost, offset, config.fetchSizeBytes)); + long endoffset = offset; + for(Message msg: msgs) { + emit(config, attempt, collector, msg); + endoffset += MessageSet.entrySize(msg); + } + BatchMeta newMeta = new BatchMeta(); + newMeta.offset = offset; + newMeta.nextOffset = endoffset; + return newMeta; + } + + public static void emit(KafkaConfig config, TransactionAttempt attempt, BatchOutputCollector collector, Message msg) { + List values = config.scheme.deserialize(Utils.toByteArray(msg.payload())); + List toEmit = new ArrayList(); + toEmit.add(attempt); + toEmit.addAll(values); + collector.emit(toEmit); + } +} diff --git a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java new file mode 100644 index 0000000000..c1ad0ec830 --- /dev/null +++ b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java @@ -0,0 +1,66 @@ +package storm.kafka; + +import backtype.storm.coordination.BatchOutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.transactional.TransactionAttempt; +import backtype.storm.transactional.partitioned.IOpaquePartitionedTransactionalSpout; +import backtype.storm.tuple.Fields; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import kafka.javaapi.consumer.SimpleConsumer; + + +public class OpaqueTransactionalKafkaSpout implements IOpaquePartitionedTransactionalSpout { + public static final String ATTEMPT_FIELD = OpaqueTransactionalKafkaSpout.class.getCanonicalName() + "/attempt"; + + KafkaConfig _config; + + public OpaqueTransactionalKafkaSpout(KafkaConfig config) { + _config = config; + } + + @Override + public IOpaquePartitionedTransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) { + return new Emitter(); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + List fields = new ArrayList(_config.scheme.getOutputFields().toList()); + fields.add(0, ATTEMPT_FIELD); + declarer.declare(new Fields(fields)); + } + + @Override + public Map getComponentConfiguration() { + backtype.storm.Config conf = new backtype.storm.Config(); + conf.registerSerialization(BatchMeta.class); + return conf; + } + + class Emitter implements IOpaquePartitionedTransactionalSpout.Emitter { + KafkaPartitionConnections _connections; + + public Emitter() { + _connections = new KafkaPartitionConnections(_config); + } + + @Override + public BatchMeta emitPartitionBatch(TransactionAttempt attempt, BatchOutputCollector collector, int partition, BatchMeta lastMeta) { + SimpleConsumer consumer = _connections.getConsumer(partition); + return KafkaUtils.emitPartitionBatchNew(_config, partition, consumer, attempt, collector, lastMeta); + } + + @Override + public int numPartitions() { + return _config.hosts.size() * _config.partitionsPerHost; + } + + @Override + public void close() { + _connections.close(); + } + } +} diff --git a/src/jvm/storm/kafka/StringScheme.java b/src/jvm/storm/kafka/StringScheme.java new file mode 100644 index 0000000000..ea4fd73650 --- /dev/null +++ b/src/jvm/storm/kafka/StringScheme.java @@ -0,0 +1,22 @@ +package storm.kafka; + +import backtype.storm.spout.Scheme; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Values; +import java.io.UnsupportedEncodingException; +import java.util.List; + +public class StringScheme implements Scheme { + + public List deserialize(byte[] bytes) { + try { + return new Values(new String(bytes, "UTF-8")); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + public Fields getOutputFields() { + return new Fields("str"); + } +} \ No newline at end of file diff --git a/src/jvm/storm/kafka/TestTopology.java b/src/jvm/storm/kafka/TestTopology.java new file mode 100644 index 0000000000..4ba5f939f0 --- /dev/null +++ b/src/jvm/storm/kafka/TestTopology.java @@ -0,0 +1,26 @@ +package storm.kafka; + +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.transactional.TransactionalTopologyBuilder; +import java.util.ArrayList; +import java.util.List; + +public class TestTopology { + public static void main(String [] args) throws Exception { + List hosts = new ArrayList(); + hosts.add("localhost"); + KafkaConfig kafkaConf = new KafkaConfig(hosts, 3, "test"); + kafkaConf.scheme = new StringScheme(); + LocalCluster cluster = new LocalCluster(); + TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("id", "spout", + new TransactionalKafkaSpout(kafkaConf), 1); + + Config config = new Config(); + config.setDebug(true); + + cluster.submitTopology("kafka-test", config, builder.buildTopology()); + + Thread.sleep(600000); + } +} diff --git a/src/jvm/storm/kafka/TransactionalKafkaSpout.java b/src/jvm/storm/kafka/TransactionalKafkaSpout.java index df20f774ce..bfadfa0a07 100644 --- a/src/jvm/storm/kafka/TransactionalKafkaSpout.java +++ b/src/jvm/storm/kafka/TransactionalKafkaSpout.java @@ -1,7 +1,5 @@ package storm.kafka; -import backtype.storm.spout.RawScheme; -import backtype.storm.spout.Scheme; import backtype.storm.task.TopologyContext; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BasePartitionedTransactionalSpout; @@ -10,7 +8,6 @@ import backtype.storm.coordination.BatchOutputCollector; import backtype.storm.tuple.Fields; import backtype.storm.utils.Utils; -import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -25,26 +22,9 @@ public class TransactionalKafkaSpout extends BasePartitionedTransactionalSpout { public static final String ATTEMPT_FIELD = TransactionalKafkaSpout.class.getCanonicalName() + "/attempt"; - public static class Config implements Serializable { - public List hosts; - public int port = 9092; - public int partitionsPerHost; - public int fetchSizeBytes = 1024*1024; - public int socketTimeoutMs = 10000; - public int bufferSizeBytes = 1024*1024; - public Scheme scheme = new RawScheme(); - public String topic; - - public Config(List hosts, int partitionsPerHost, String topic) { - this.hosts = hosts; - this.partitionsPerHost = partitionsPerHost; - this.topic = topic; - } - } - - Config _config; + KafkaConfig _config; - public TransactionalKafkaSpout(Config config) { + public TransactionalKafkaSpout(KafkaConfig config) { _config = config; } @@ -60,32 +40,22 @@ public void close() { } class Emitter implements IPartitionedTransactionalSpout.Emitter { - Map _kafka = new HashMap(); + KafkaPartitionConnections _connections; + + public Emitter() { + _connections = new KafkaPartitionConnections(_config); + } @Override public BatchMeta emitPartitionBatchNew(TransactionAttempt attempt, BatchOutputCollector collector, int partition, BatchMeta lastMeta) { - SimpleConsumer consumer = connect(partition); + SimpleConsumer consumer = _connections.getConsumer(partition); - long offset = 0; - if(lastMeta!=null) { - offset = lastMeta.nextOffset; - } - - ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition % _config.partitionsPerHost, offset, _config.fetchSizeBytes)); - long endoffset = offset; - for(Message msg: msgs) { - emit(attempt, collector, msg); - endoffset += MessageSet.entrySize(msg); - } - BatchMeta newMeta = new BatchMeta(); - newMeta.offset = offset; - newMeta.nextOffset = endoffset; - return newMeta; + return KafkaUtils.emitPartitionBatchNew(_config, partition, consumer, attempt, collector, lastMeta); } @Override public void emitPartitionBatch(TransactionAttempt attempt, BatchOutputCollector collector, int partition, BatchMeta meta) { - SimpleConsumer consumer = connect(partition); + SimpleConsumer consumer = _connections.getConsumer(partition); ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition % _config.partitionsPerHost, meta.offset, _config.fetchSizeBytes)); long currOffset = meta.offset; @@ -94,33 +64,15 @@ public void emitPartitionBatch(TransactionAttempt attempt, BatchOutputCollector if(currOffset > meta.nextOffset) { throw new RuntimeException("Error when re-emitting batch. overshot the end offset"); } - emit(attempt, collector, msg); + KafkaUtils.emit(_config, attempt, collector, msg); currOffset += MessageSet.entrySize(msg); } } - private void emit(TransactionAttempt attempt, BatchOutputCollector collector, Message msg) { - List values = _config.scheme.deserialize(Utils.toByteArray(msg.payload())); - List toEmit = new ArrayList(); - toEmit.add(attempt); - toEmit.addAll(values); - collector.emit(toEmit); - } - - private SimpleConsumer connect(int partition) { - if(!_kafka.containsKey(partition)) { - int hostIndex = partition % _config.hosts.size(); - _kafka.put(partition, new SimpleConsumer(_config.hosts.get(hostIndex), _config.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); - } - return _kafka.get(partition); - } - @Override public void close() { - for(SimpleConsumer consumer: _kafka.values()) { - consumer.close(); - } + _connections.close(); } } From 0f96d15add499fde4bad155be7d122ab52094fcc Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 11 Feb 2012 19:48:32 -0800 Subject: [PATCH 051/349] finished opaquetransactionalkafkaspout --- src/jvm/storm/kafka/FailedFetchException.java | 7 ++++++ src/jvm/storm/kafka/KafkaUtils.java | 15 +++++++++++-- .../kafka/OpaqueTransactionalKafkaSpout.java | 19 ++++++++++++++-- src/jvm/storm/kafka/TestTopology.java | 22 ++++++++++++++++--- 4 files changed, 56 insertions(+), 7 deletions(-) create mode 100644 src/jvm/storm/kafka/FailedFetchException.java diff --git a/src/jvm/storm/kafka/FailedFetchException.java b/src/jvm/storm/kafka/FailedFetchException.java new file mode 100644 index 0000000000..0a3b4b8a57 --- /dev/null +++ b/src/jvm/storm/kafka/FailedFetchException.java @@ -0,0 +1,7 @@ +package storm.kafka; + +public class FailedFetchException extends RuntimeException { + public FailedFetchException(Exception e) { + super(e); + } +} diff --git a/src/jvm/storm/kafka/KafkaUtils.java b/src/jvm/storm/kafka/KafkaUtils.java index 81d8f130ef..a5ddd8a189 100644 --- a/src/jvm/storm/kafka/KafkaUtils.java +++ b/src/jvm/storm/kafka/KafkaUtils.java @@ -3,6 +3,7 @@ import backtype.storm.coordination.BatchOutputCollector; import backtype.storm.transactional.TransactionAttempt; import backtype.storm.utils.Utils; +import java.net.ConnectException; import java.util.ArrayList; import java.util.List; import kafka.api.FetchRequest; @@ -12,13 +13,23 @@ import kafka.message.MessageSet; public class KafkaUtils { + + public static BatchMeta emitPartitionBatchNew(KafkaConfig config, int partition, SimpleConsumer consumer, TransactionAttempt attempt, BatchOutputCollector collector, BatchMeta lastMeta) { long offset = 0; if(lastMeta!=null) { offset = lastMeta.nextOffset; } - - ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(config.topic, partition % config.partitionsPerHost, offset, config.fetchSizeBytes)); + ByteBufferMessageSet msgs; + try { + msgs = consumer.fetch(new FetchRequest(config.topic, partition % config.partitionsPerHost, offset, config.fetchSizeBytes)); + } catch(Exception e) { + if(e instanceof ConnectException) { + throw new FailedFetchException(e); + } else { + throw new RuntimeException(e); + } + } long endoffset = offset; for(Message msg: msgs) { emit(config, attempt, collector, msg); diff --git a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java index c1ad0ec830..57be9a1efe 100644 --- a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java +++ b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java @@ -10,9 +10,12 @@ import java.util.List; import java.util.Map; import kafka.javaapi.consumer.SimpleConsumer; +import org.apache.log4j.Logger; public class OpaqueTransactionalKafkaSpout implements IOpaquePartitionedTransactionalSpout { + public static final Logger LOG = Logger.getLogger(OpaqueTransactionalKafkaSpout.class); + public static final String ATTEMPT_FIELD = OpaqueTransactionalKafkaSpout.class.getCanonicalName() + "/attempt"; KafkaConfig _config; @@ -49,8 +52,20 @@ public Emitter() { @Override public BatchMeta emitPartitionBatch(TransactionAttempt attempt, BatchOutputCollector collector, int partition, BatchMeta lastMeta) { - SimpleConsumer consumer = _connections.getConsumer(partition); - return KafkaUtils.emitPartitionBatchNew(_config, partition, consumer, attempt, collector, lastMeta); + try { + SimpleConsumer consumer = _connections.getConsumer(partition); + return KafkaUtils.emitPartitionBatchNew(_config, partition, consumer, attempt, collector, lastMeta); + } catch(FailedFetchException e) { + LOG.warn("Failed to fetch from partition " + partition); + if(lastMeta==null) { + return null; + } else { + BatchMeta ret = new BatchMeta(); + ret.offset = lastMeta.nextOffset; + ret.nextOffset = lastMeta.nextOffset; + return ret; + } + } } @Override diff --git a/src/jvm/storm/kafka/TestTopology.java b/src/jvm/storm/kafka/TestTopology.java index 4ba5f939f0..65069fdcd6 100644 --- a/src/jvm/storm/kafka/TestTopology.java +++ b/src/jvm/storm/kafka/TestTopology.java @@ -2,11 +2,27 @@ import backtype.storm.Config; import backtype.storm.LocalCluster; +import backtype.storm.topology.BasicOutputCollector; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseBasicBolt; import backtype.storm.transactional.TransactionalTopologyBuilder; +import backtype.storm.tuple.Tuple; import java.util.ArrayList; import java.util.List; public class TestTopology { + public static class PrinterBolt extends BaseBasicBolt { + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + System.out.println(tuple); + } + + } + public static void main(String [] args) throws Exception { List hosts = new ArrayList(); hosts.add("localhost"); @@ -14,10 +30,10 @@ public static void main(String [] args) throws Exception { kafkaConf.scheme = new StringScheme(); LocalCluster cluster = new LocalCluster(); TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("id", "spout", - new TransactionalKafkaSpout(kafkaConf), 1); - + new OpaqueTransactionalKafkaSpout(kafkaConf), 1); + builder.setBolt("printer", new PrinterBolt()) + .shuffleGrouping("spout"); Config config = new Config(); - config.setDebug(true); cluster.submitTopology("kafka-test", config, builder.buildTopology()); From f19334e13c7566e86f3ebf370b1828e203a2461b Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 22 Feb 2012 21:50:36 -0800 Subject: [PATCH 052/349] update storm dep --- m2-pom.xml | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/m2-pom.xml b/m2-pom.xml index 351a98dac9..4d79ba479a 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -38,7 +38,7 @@ storm storm - 0.6.2 + 0.7.0-rc provided diff --git a/project.clj b/project.clj index f9a8b511ea..841388f9c2 100644 --- a/project.clj +++ b/project.clj @@ -15,6 +15,6 @@ [org.twitter4j/twitter4j-stream "2.2.5-SNAPSHOT"] ] - :dev-dependencies [[storm "0.7.0-SNAPSHOT"] + :dev-dependencies [[storm "0.7.0-rc"] ]) From 159f41b5064c38d132146de97c31f5ae1f3df231 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 27 Feb 2012 14:52:11 -0800 Subject: [PATCH 053/349] bump dep to 0.7.0 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 841388f9c2..3469e5938a 100644 --- a/project.clj +++ b/project.clj @@ -15,6 +15,6 @@ [org.twitter4j/twitter4j-stream "2.2.5-SNAPSHOT"] ] - :dev-dependencies [[storm "0.7.0-rc"] + :dev-dependencies [[storm "0.7.0"] ]) From 50a021bddbadbe16e064034b43fbb99d09240176 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 27 Feb 2012 14:56:30 -0800 Subject: [PATCH 054/349] update dep to storm 0.7.0 --- project.clj | 2 +- src/jvm/storm/kafka/TestTopology.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 314fbd4ada..e0ed77c1ea 100644 --- a/project.clj +++ b/project.clj @@ -5,6 +5,6 @@ :dependencies [[storm/kafka "0.6.0" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.7.0-rc2-SNAPSHOT"]] + :dev-dependencies [[storm "0.7.0"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) diff --git a/src/jvm/storm/kafka/TestTopology.java b/src/jvm/storm/kafka/TestTopology.java index 65069fdcd6..01155de2ac 100644 --- a/src/jvm/storm/kafka/TestTopology.java +++ b/src/jvm/storm/kafka/TestTopology.java @@ -18,7 +18,7 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { @Override public void execute(Tuple tuple, BasicOutputCollector collector) { - System.out.println(tuple); + System.out.println(tuple.toString()); } } From 3ebbf278b0e85bdb4c8fac4ebb9099853b214c74 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 27 Feb 2012 14:57:33 -0800 Subject: [PATCH 055/349] update readme --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 5f18b258d0..e35c9cb282 100644 --- a/README.markdown +++ b/README.markdown @@ -1,4 +1,4 @@ -storm-kafka provides a TransactionalSpout implementation for Apache Kafka. TransactionalSpout won't be available in Storm until 0.7.0 (a development release is currently available though). This implementation has only been lightly tested. +storm-kafka provides a TransactionalSpout implementation for Apache Kafka. The master branch is compatible with Kafka 0.6, the kafka0.7 branch is compatible with Kafka 0.7. From d281d285eaec66b73c3ef0d2070807aaedc04861 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 27 Feb 2012 15:21:55 -0800 Subject: [PATCH 056/349] include clojure as a dev dep --- project.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project.clj b/project.clj index e0ed77c1ea..f21d88231d 100644 --- a/project.clj +++ b/project.clj @@ -5,6 +5,7 @@ :dependencies [[storm/kafka "0.6.0" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.7.0"]] + :dev-dependencies [[storm "0.7.0"] + [org.clojure/clojure "1.2.0"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) From e54cd99625bc15876a794c6f23eb7671ec3b288e Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 29 Feb 2012 12:25:56 -0800 Subject: [PATCH 057/349] update maven instructions --- README.markdown | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.markdown b/README.markdown index 08a235df5a..6815dc3639 100644 --- a/README.markdown +++ b/README.markdown @@ -18,8 +18,16 @@ java -cp `lein classpath` storm.starter.ExclamationTopology ## Maven -Maven is an alternative to Leiningen. storm-starter contains m2-pom.xml which can be used with Maven using the -f option, e.g.: +Maven is an alternative to Leiningen. storm-starter contains m2-pom.xml which can be used with Maven using the -f option. For example, to compile and run `WordCountTopology` in local mode, use this command: + +``` +mvn -f m2-pom.xml compile exec:java -Dexec.classpathScope=compile -Dexec.mainClass=storm.starter.WordCountTopology +``` + +You can package a jar suitable for submitting to a cluster with this command: ``` mvn -f m2-pom.xml package -``` \ No newline at end of file +``` + +This will package your code and all the non-Storm dependencies into a single "uberjar" at the path `target/storm-starter-{version}-jar-with-dependencies.jar`. From a94a6cf7d406119322dae22e34ad0b2225e48625 Mon Sep 17 00:00:00 2001 From: millerjam Date: Fri, 16 Mar 2012 23:27:40 -0400 Subject: [PATCH 058/349] bump dependency in pom to 0.7.0 --- m2-pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/m2-pom.xml b/m2-pom.xml index 4d79ba479a..282e97d5f6 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -38,7 +38,7 @@ storm storm - 0.7.0-rc + 0.7.0 provided From 6b1384b8c584df639fd6b5ac25627ee9a3b1bfd6 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 22 Mar 2012 21:25:03 -0700 Subject: [PATCH 059/349] fix casting in SingleJoinBolt --- src/jvm/storm/starter/bolt/SingleJoinBolt.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jvm/storm/starter/bolt/SingleJoinBolt.java b/src/jvm/storm/starter/bolt/SingleJoinBolt.java index e83b9be196..c353b7a2e2 100644 --- a/src/jvm/storm/starter/bolt/SingleJoinBolt.java +++ b/src/jvm/storm/starter/bolt/SingleJoinBolt.java @@ -32,8 +32,8 @@ public SingleJoinBolt(Fields outFields) { public void prepare(Map conf, TopologyContext context, OutputCollector collector) { _fieldLocations = new HashMap(); _collector = collector; - long timeout = (Long) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS); - _pending = new TimeCacheMap, Map>((int) timeout, new ExpireCallback()); + int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue(); + _pending = new TimeCacheMap, Map>(timeout, new ExpireCallback()); _numSources = context.getThisSources().size(); Set idFields = null; for(GlobalStreamId source: context.getThisSources().keySet()) { From c1b215a077127704b7f06a6533cbd4443d0c8b25 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 30 Mar 2012 00:28:00 -0700 Subject: [PATCH 060/349] leiningen installation instructions --- README.markdown | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.markdown b/README.markdown index 6815dc3639..c7b1052381 100644 --- a/README.markdown +++ b/README.markdown @@ -10,6 +10,8 @@ More information about Storm can be found on the [project page](http://github.co ## Running an example with Leiningen +Install Leiningen by following the installation instructions [here](https://github.com/technomancy/leiningen). + ``` lein deps lein compile From 96d442dc9ad680c6e34aed4b09a79b6e402c0f62 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 2 Apr 2012 13:05:06 -0700 Subject: [PATCH 061/349] 0.7.1 compatible --- .../storm/kafka/OpaqueTransactionalKafkaSpout.java | 13 +++++++++++++ src/jvm/storm/kafka/TransactionalKafkaSpout.java | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java index 57be9a1efe..45dff4886b 100644 --- a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java +++ b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java @@ -28,6 +28,11 @@ public OpaqueTransactionalKafkaSpout(KafkaConfig config) { public IOpaquePartitionedTransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) { return new Emitter(); } + + @Override + public IOpaquePartitionedTransactionalSpout.Coordinator getCoordinator(Map map, TopologyContext tc) { + throw new UnsupportedOperationException("Not supported yet."); + } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { @@ -43,6 +48,14 @@ public Map getComponentConfiguration() { return conf; } + class Coordinator implements IOpaquePartitionedTransactionalSpout.Coordinator { + @Override + public boolean isReady() { + //TODO: can do a more sophisticated strategy by looking at the high water marks for each partition + return true; + } + } + class Emitter implements IOpaquePartitionedTransactionalSpout.Emitter { KafkaPartitionConnections _connections; diff --git a/src/jvm/storm/kafka/TransactionalKafkaSpout.java b/src/jvm/storm/kafka/TransactionalKafkaSpout.java index bfadfa0a07..876bc97a56 100644 --- a/src/jvm/storm/kafka/TransactionalKafkaSpout.java +++ b/src/jvm/storm/kafka/TransactionalKafkaSpout.java @@ -37,6 +37,12 @@ public int numPartitions() { @Override public void close() { } + + @Override + public boolean isReady() { + //TODO: can do a more sophisticated strategy by looking at the high water marks for each partition + return true; + } } class Emitter implements IPartitionedTransactionalSpout.Emitter { From f4c0886a424ce730ef8553f3bd5271bf3637ac91 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 2 Apr 2012 13:05:22 -0700 Subject: [PATCH 062/349] 0.7.1-SNAPSHOT --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index f21d88231d..37d0c6316a 100644 --- a/project.clj +++ b/project.clj @@ -1,11 +1,11 @@ -(defproject storm/storm-kafka "0.7.0-SNAPSHOT" +(defproject storm/storm-kafka "0.7.1-SNAPSHOT" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} :dependencies [[storm/kafka "0.6.0" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.7.0"] + :dev-dependencies [[storm "0.7.1"] [org.clojure/clojure "1.2.0"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) From e49662519c3e697845db33c7c535df2095fc13f4 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 2 Apr 2012 13:05:06 -0700 Subject: [PATCH 063/349] 0.7.1 compatible --- .../storm/kafka/OpaqueTransactionalKafkaSpout.java | 13 +++++++++++++ src/jvm/storm/kafka/TransactionalKafkaSpout.java | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java index 57be9a1efe..45dff4886b 100644 --- a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java +++ b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java @@ -28,6 +28,11 @@ public OpaqueTransactionalKafkaSpout(KafkaConfig config) { public IOpaquePartitionedTransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) { return new Emitter(); } + + @Override + public IOpaquePartitionedTransactionalSpout.Coordinator getCoordinator(Map map, TopologyContext tc) { + throw new UnsupportedOperationException("Not supported yet."); + } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { @@ -43,6 +48,14 @@ public Map getComponentConfiguration() { return conf; } + class Coordinator implements IOpaquePartitionedTransactionalSpout.Coordinator { + @Override + public boolean isReady() { + //TODO: can do a more sophisticated strategy by looking at the high water marks for each partition + return true; + } + } + class Emitter implements IOpaquePartitionedTransactionalSpout.Emitter { KafkaPartitionConnections _connections; diff --git a/src/jvm/storm/kafka/TransactionalKafkaSpout.java b/src/jvm/storm/kafka/TransactionalKafkaSpout.java index cc11d2360d..e28e898f53 100644 --- a/src/jvm/storm/kafka/TransactionalKafkaSpout.java +++ b/src/jvm/storm/kafka/TransactionalKafkaSpout.java @@ -34,6 +34,12 @@ public int numPartitions() { @Override public void close() { } + + @Override + public boolean isReady() { + //TODO: can do a more sophisticated strategy by looking at the high water marks for each partition + return true; + } } class Emitter implements IPartitionedTransactionalSpout.Emitter { From c1e5112d7459b890bc70816e3a26a98b37f4e72a Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 2 Apr 2012 13:08:07 -0700 Subject: [PATCH 064/349] storm 0.7.1 compatible --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 7f26ad635f..fe30601392 100644 --- a/project.clj +++ b/project.clj @@ -1,11 +1,11 @@ -(defproject storm/storm-kafka "0.7.0-kafka7-SNAPSHOT" +(defproject storm/storm-kafka "0.7.1-kafka7-SNAPSHOT" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} :dependencies [[storm/kafka "0.7.0-incubating" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.7.0"] + :dev-dependencies [[storm "0.7.1"] [org.clojure/clojure "1.2.0"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) From c0118e3e32f2e5e0477cf27a0566b2df37750a71 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Tue, 3 Apr 2012 14:20:15 -0700 Subject: [PATCH 065/349] upgrade to 0.7.1 --- multilang/resources/storm.py | 173 ++++++++++++++++++++++++----------- multilang/resources/storm.rb | 131 +++++++++++++++++++------- project.clj | 2 +- 3 files changed, 219 insertions(+), 87 deletions(-) diff --git a/multilang/resources/storm.py b/multilang/resources/storm.py index 6bb6d04f12..c06b255733 100644 --- a/multilang/resources/storm.py +++ b/multilang/resources/storm.py @@ -1,50 +1,88 @@ import sys import os import traceback +from collections import deque try: - import cjson - json_encode = cjson.encode - json_decode = lambda x: cjson.decode(x, all_unicode=True) + import simplejson as json except ImportError: import json - json_encode = lambda x: json.dumps(x, ensure_ascii=False) - json_decode = lambda x: json.loads(unicode(x)) -def readStringMsg(): +json_encode = lambda x: json.dumps(x) +json_decode = lambda x: json.loads(x) + +#reads lines and reconstructs newlines appropriately +def readMsg(): msg = "" while True: line = sys.stdin.readline()[0:-1] if line == "end": break msg = msg + line + "\n" - return msg[0:-1] + return json_decode(msg[0:-1]) +MODE = None ANCHOR_TUPLE = None -#reads lines and reconstructs newlines appropriately -def readMsg(): - return json_decode(readStringMsg()) - -def sendToParent(s): - print s +#queue up commands we read while trying to read taskids +pending_commands = deque() + +def readTaskIds(): + if pending_taskids: + return pending_taskids.popleft() + else: + msg = readMsg() + while type(msg) is not list: + pending_commands.append(msg) + msg = readMsg() + return msg + +#queue up taskids we read while trying to read commands/tuples +pending_taskids = deque() + +def readCommand(): + if pending_commands: + return pending_commands.popleft() + else: + msg = readMsg() + while type(msg) is list: + pending_taskids.append(msg) + msg = readMsg() + return msg + +def readTuple(): + cmd = readCommand() + return Tuple(cmd["id"], cmd["comp"], cmd["stream"], cmd["task"], cmd["tuple"]) + +def sendMsgToParent(msg): + print json_encode(msg) print "end" sys.stdout.flush() - + def sync(): - print "sync" - sys.stdout.flush() + sendMsgToParent({'command':'sync'}) def sendpid(heartbeatdir): pid = os.getpid() - print pid - sys.stdout.flush() + sendMsgToParent({'pid':pid}) open(heartbeatdir + "/" + str(pid), "w").close() -def sendMsgToParent(amap): - sendToParent(json_encode(amap)) +def emit(*args, **kwargs): + __emit(*args, **kwargs) + return readTaskIds() + +def emitDirect(task, *args, **kwargs): + kwargs[directTask] = task + __emit(*args, **kwargs) + +def __emit(*args, **kwargs): + global MODE + if MODE == Bolt: + emitBolt(*args, **kwargs) + elif MODE == Spout: + emitSpout(*args, **kwargs) -def emittuple(tup, stream=None, anchors = [], directTask=None): +def emitBolt(tup, stream=None, anchors = [], directTask=None): global ANCHOR_TUPLE if ANCHOR_TUPLE is not None: anchors = [ANCHOR_TUPLE] @@ -57,13 +95,16 @@ def emittuple(tup, stream=None, anchors = [], directTask=None): m["tuple"] = tup sendMsgToParent(m) -def emit(tup, stream=None, anchors = []): - emittuple(tup, stream=stream, anchors=anchors) - #read back task ids - return readMsg() - -def emitDirect(task, tup, stream=None, anchors = []): - emittuple(tup, stream=stream, anchors=anchors, directTask=task) +def emitSpout(tup, stream=None, id=None, directTask=None): + m = {"command": "emit"} + if id is not None: + m["id"] = id + if stream is not None: + m["stream"] = stream + if directTask is not None: + m["task"] = directTask + m["tuple"] = tup + sendMsgToParent(m) def ack(tup): sendMsgToParent({"command": "ack", "id": tup.id}) @@ -74,22 +115,12 @@ def fail(tup): def log(msg): sendMsgToParent({"command": "log", "msg": msg}) -# read the stormconf and context -def readenv(): - conf = readMsg() - context = readMsg() - return [conf, context] - -def readtuple(): - tupmap = readMsg() - return Tuple(tupmap["id"], tupmap["comp"], tupmap["stream"], tupmap["task"], tupmap["tuple"]) +def initComponent(): + setupInfo = readMsg() + sendpid(setupInfo['pidDir']) + return [setupInfo['conf'], setupInfo['context']] -def initbolt(): - heartbeatdir = readStringMsg() - sendpid(heartbeatdir) - return readenv() - -class Tuple: +class Tuple: def __init__(self, id, component, stream, task, values): self.id = id self.component = component @@ -97,47 +128,79 @@ def __init__(self, id, component, stream, task, values): self.task = task self.values = values + def __repr__(self): + return '<%s%s>' % ( + self.__class__.__name__, + ''.join(' %s=%r' % (k, self.__dict__[k]) for k in sorted(self.__dict__.keys()))) + class Bolt: def initialize(self, stormconf, context): pass - + def process(self, tuple): pass - + def run(self): - conf, context = initbolt() + global MODE + MODE = Bolt + conf, context = initComponent() self.initialize(conf, context) try: while True: - tup = readtuple() + tup = readTuple() self.process(tup) - sync() except Exception, e: - log(traceback.format_exc(e)) + log(traceback.format_exc(e)) class BasicBolt: def initialize(self, stormconf, context): pass - + def process(self, tuple): pass - + def run(self): + global MODE + MODE = Bolt global ANCHOR_TUPLE - conf, context = initbolt() + conf, context = initComponent() self.initialize(conf, context) try: while True: - tup = readtuple() + tup = readTuple() ANCHOR_TUPLE = tup self.process(tup) ack(tup) - sync() except Exception, e: log(traceback.format_exc(e)) class Spout: - pass + def initialize(self, conf, context): + pass + def ack(self, id): + pass + + def fail(self, id): + pass + def nextTuple(self): + pass + def run(self): + global MODE + MODE = Spout + conf, context = initComponent() + self.initialize(conf, context) + try: + while True: + msg = readCommand() + if msg["command"] == "next": + self.nextTuple() + if msg["command"] == "ack": + self.ack(msg["id"]) + if msg["command"] == "fail": + self.fail(msg["id"]) + sync() + except Exception, e: + log(traceback.format_exc(e)) diff --git a/multilang/resources/storm.rb b/multilang/resources/storm.rb index 32c17b631b..017fc25601 100644 --- a/multilang/resources/storm.rb +++ b/multilang/resources/storm.rb @@ -3,7 +3,14 @@ module Storm module Protocol - def read_string_message + class << self + attr_accessor :mode, :pending_taskids, :pending_commands + end + + self.pending_taskids = [] + self.pending_commands = [] + + def read_message msg = "" loop do line = STDIN.readline.chomp @@ -11,45 +18,80 @@ def read_string_message msg << line msg << "\n" end - msg.chomp + JSON.parse msg.chomp + end + + def read_task_ids + Storm::Protocol.pending_taskids.shift || + begin + msg = read_message + until msg.is_a? Array + Storm::Protocol.pending_commands.push(msg) + msg = read_message + end + msg + end end - def read_message - JSON.parse read_string_message + def read_command + Storm::Protocol.pending_commands.shift || + begin + msg = read_message + while msg.is_a? Array + Storm::Protocol.pending_taskids.push(msg) + msg = read_message + end + msg + end end - def send_to_parent(s) - puts s + def send_msg_to_parent(msg) + puts msg.to_json puts "end" STDOUT.flush end def sync - puts "sync" - STDOUT.flush + send_msg_to_parent({'command' => 'sync'}) end def send_pid(heartbeat_dir) pid = Process.pid - puts pid - STDOUT.flush + send_msg_to_parent({'pid' => pid}) File.open("#{heartbeat_dir}/#{pid}", "w").close end - def send_msg_to_parent(hash) - send_to_parent hash.to_json + def emit_bolt(tup, args = {}) + stream = args[:stream] + anchors = args[:anchors] || args[:anchor] || [] + anchors = [anchors] unless anchors.is_a? Enumerable + direct = args[:direct_task] + m = {:command => :emit, :anchors => anchors.map(&:id), :tuple => tup} + m[:stream] = stream if stream + m[:task] = direct if direct + send_msg_to_parent m + read_task_ids unless direct end - def emit_tuple(tup, stream = nil, anchors = [], direct = nil) - m = {:command => :emit, :anchors => anchors.map(&:id), :tuple => tup} + def emit_spout(tup, args = {}) + stream = args[:stream] + id = args[:id] + direct = args[:direct_task] + m = {:command => :emit, :tuple => tup} + m[:id] = id if id m[:stream] = stream if stream m[:task] = direct if direct send_msg_to_parent m + read_task_ids unless direct end - def emit(tup, stream = nil, anchors = [], direct = nil) - emit_tuple tup, stream, anchors, direct - read_message + def emit(*args) + case Storm::Protocol.mode + when 'spout' + emit_spout(*args) + when 'bolt' + emit_bolt(*args) + end end def ack(tup) @@ -64,8 +106,10 @@ def log(msg) send_msg_to_parent :command => :log, :msg => msg.to_s end - def read_env - [read_message, read_message] + def handshake + setup_info = read_message + send_pid setup_info['pidDir'] + [setup_info['conf'], setup_info['context']] end end @@ -88,29 +132,54 @@ def self.from_hash(hash) class Bolt include Storm::Protocol - def initialize(conf = nil, context = nil) - @conf = conf - @context = context - end + def prepare(conf, context); end - def process(tuple) - end + def process(tuple); end def run - heartbeat_dir = read_string_message - send_pid heartbeat_dir - @conf, @context = read_env + Storm::Protocol.mode = 'bolt' + prepare(*handshake) begin while true - process Tuple.from_hash(read_message) - sync + process Tuple.from_hash(read_command) end rescue Exception => e - log e + log 'Exception in bolt: ' + e.message + ' - ' + e.backtrace.join('\n') end end end class Spout + include Storm::Protocol + + def open(conf, context); end + + def nextTuple; end + + def ack(id); end + + def fail(id); end + + def run + Storm::Protocol.mode = 'spout' + open(*handshake) + + begin + while true + msg = read_command + case msg['command'] + when 'next' + nextTuple + when 'ack' + ack(msg['id']) + when 'fail' + fail(msg['id']) + end + sync + end + rescue Exception => e + log 'Exception in spout: ' + e.message + ' - ' + e.backtrace.join('\n') + end + end end end diff --git a/project.clj b/project.clj index 3469e5938a..bbc80147b0 100644 --- a/project.clj +++ b/project.clj @@ -15,6 +15,6 @@ [org.twitter4j/twitter4j-stream "2.2.5-SNAPSHOT"] ] - :dev-dependencies [[storm "0.7.0"] + :dev-dependencies [[storm "0.7.1"] ]) From d1157e6ac0ae5b2be1ee26713265eae204f4ac23 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sun, 8 Apr 2012 20:29:01 -0400 Subject: [PATCH 066/349] fix opaquepartitionedtransactionalspotu --- src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java index 45dff4886b..f9e08cf726 100644 --- a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java +++ b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java @@ -31,7 +31,7 @@ public IOpaquePartitionedTransactionalSpout.Emitter getEmitter(Map co @Override public IOpaquePartitionedTransactionalSpout.Coordinator getCoordinator(Map map, TopologyContext tc) { - throw new UnsupportedOperationException("Not supported yet."); + return new Coordinator(); } @Override From f6d464f2a02cf838e1758812239432d562683bed Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sun, 8 Apr 2012 20:29:01 -0400 Subject: [PATCH 067/349] fix opaquepartitionedtransactionalspotu --- src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java index 45dff4886b..f9e08cf726 100644 --- a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java +++ b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java @@ -31,7 +31,7 @@ public IOpaquePartitionedTransactionalSpout.Emitter getEmitter(Map co @Override public IOpaquePartitionedTransactionalSpout.Coordinator getCoordinator(Map map, TopologyContext tc) { - throw new UnsupportedOperationException("Not supported yet."); + return new Coordinator(); } @Override From 3ee0a8186d4c215ff2e25098bbf9afc45df61c8b Mon Sep 17 00:00:00 2001 From: Paul O'Fallon Date: Mon, 9 Apr 2012 17:26:25 -0300 Subject: [PATCH 068/349] update POM to Storm 0.7.1 --- m2-pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/m2-pom.xml b/m2-pom.xml index 282e97d5f6..192f97869d 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -38,7 +38,7 @@ storm storm - 0.7.0 + 0.7.1 provided From 203c48445f33b9bcbdbc377f3e6311babca100fd Mon Sep 17 00:00:00 2001 From: Andrew Otto Date: Wed, 11 Apr 2012 12:42:20 -0400 Subject: [PATCH 069/349] Fixing bug where COUNT_DATABASE was not updated with newVal in KeyedCountUpdater.finishBatch --- src/jvm/storm/starter/TransactionalWords.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jvm/storm/starter/TransactionalWords.java b/src/jvm/storm/starter/TransactionalWords.java index 674600f660..0252b66134 100644 --- a/src/jvm/storm/starter/TransactionalWords.java +++ b/src/jvm/storm/starter/TransactionalWords.java @@ -114,7 +114,7 @@ public void finishBatch() { newVal.count = val.count; } newVal.count = newVal.count + _counts.get(key); - COUNT_DATABASE.put(key, val); + COUNT_DATABASE.put(key, newVal); } else { newVal = val; } From 55362f5a918a7fcc4b94bdd13448ded3a4fdc1ef Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 28 Apr 2012 01:36:30 -0700 Subject: [PATCH 070/349] fix problem with partition -> local partition mapping, added regular kafka spout --- .gitignore | 1 + project.clj | 3 +- .../kafka/KafkaPartitionConnections.java | 12 +- src/jvm/storm/kafka/KafkaSpout.java | 226 ++++++++++++++++++ src/jvm/storm/kafka/KafkaUtils.java | 1 - src/jvm/storm/kafka/SpoutConfig.java | 18 ++ 6 files changed, 255 insertions(+), 6 deletions(-) create mode 100644 src/jvm/storm/kafka/KafkaSpout.java create mode 100644 src/jvm/storm/kafka/SpoutConfig.java diff --git a/.gitignore b/.gitignore index ab43f1220e..39a95fcda1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ classes/ lib/ +pom.xml diff --git a/project.clj b/project.clj index f59b4b428f..889e7c2fba 100644 --- a/project.clj +++ b/project.clj @@ -6,6 +6,7 @@ :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] :dev-dependencies [[storm "0.7.1"] - [org.clojure/clojure "1.2.0"]] + [org.clojure/clojure "1.2.0"] + [org.clojure/clojure-contrib "1.2.0"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) diff --git a/src/jvm/storm/kafka/KafkaPartitionConnections.java b/src/jvm/storm/kafka/KafkaPartitionConnections.java index 25199f42b4..218f399d3f 100644 --- a/src/jvm/storm/kafka/KafkaPartitionConnections.java +++ b/src/jvm/storm/kafka/KafkaPartitionConnections.java @@ -14,12 +14,16 @@ public KafkaPartitionConnections(KafkaConfig conf) { } public SimpleConsumer getConsumer(int partition) { - if(!_kafka.containsKey(partition)) { - int hostIndex = partition % _config.hosts.size(); - _kafka.put(partition, new SimpleConsumer(_config.hosts.get(hostIndex), _config.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); + int hostIndex = partition / _config.partitionsPerHost; + if(!_kafka.containsKey(hostIndex)) { + _kafka.put(hostIndex, new SimpleConsumer(_config.hosts.get(hostIndex), _config.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); } - return _kafka.get(partition); + return _kafka.get(hostIndex); + } + + public int getHostPartition(int globalPartition) { + return globalPartition % _config.partitionsPerHost; } public void close() { diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java new file mode 100644 index 0000000000..bd81efc378 --- /dev/null +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -0,0 +1,226 @@ +package storm.kafka; + +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.spout.SpoutOutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.TopologyBuilder; +import backtype.storm.topology.base.BaseRichSpout; +import backtype.storm.transactional.state.TransactionalState; +import backtype.storm.utils.Utils; +import java.util.ArrayList; +import kafka.javaapi.consumer.SimpleConsumer; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; +import kafka.javaapi.message.ByteBufferMessageSet; +import kafka.message.MessageAndOffset; +import kafka.api.FetchRequest; + + +public class KafkaSpout extends BaseRichSpout { + static class KafkaMessageId { + public int partition; + public long offset; + + public KafkaMessageId(int partition, long offset) { + this.partition = partition; + this.offset = offset; + } + } + + protected class PartitionManager { + Long _emittedToOffset; + SortedSet _pending = new TreeSet(); + Long _committedTo; + int _partition; + LinkedList _waitingToEmit = new LinkedList(); + + public PartitionManager(int partition) { + _partition = partition; + _committedTo = (Long) _state.getData(committedPath()); + if(_committedTo==null) _committedTo = 0L; + _emittedToOffset = _committedTo; + } + + public boolean next() { + if(_waitingToEmit.isEmpty()) fill(); + MessageAndOffset toEmit = _waitingToEmit.pollFirst(); + if(toEmit==null) return false; + List tup = _kafkaConfig.scheme.deserialize(Utils.toByteArray(toEmit.message().payload())); + _collector.emit(tup, new KafkaMessageId(_partition, actualOffset(toEmit))); + return true; + } + + private void fill() { + SimpleConsumer consumer = _partitions.getConsumer(_partition); + ByteBufferMessageSet msgs = consumer.fetch( + new FetchRequest( + _kafkaConfig.topic, + _partitions.getHostPartition(_partition), + _emittedToOffset, + _kafkaConfig.fetchSizeBytes)); + for(MessageAndOffset msg: msgs) { + _pending.add(actualOffset(msg)); + _waitingToEmit.add(msg); + _emittedToOffset = msg.offset(); + } + } + + public void ack(Long offset) { + _pending.remove(offset); + } + + public void fail(Long offset) { + //TODO: should it use in-memory ack set to skip anything that's been acked but not committed??? + // things might get crazy with lots of timeouts + if(_emittedToOffset > offset) { + _emittedToOffset = offset; + _pending.tailSet(offset).clear(); + } + } + + public void commit() { + long committedTo; + if(_pending.isEmpty()) { + committedTo = _emittedToOffset; + } else { + committedTo = _pending.first(); + } + if(committedTo!=_committedTo) { + _state.setData(committedPath(), committedTo); + _committedTo = committedTo; + } + } + + private long actualOffset(MessageAndOffset msg) { + return msg.offset() - msg.message().serializedSize(); + + } + + private String committedPath() { + return _spoutConfig.id + "/" + _partition; + } + } + + KafkaConfig _kafkaConfig; + SpoutConfig _spoutConfig; + SpoutOutputCollector _collector; + TransactionalState _state; + KafkaPartitionConnections _partitions; + Map _managers = new HashMap(); + + long _lastUpdateMs = 0; + + int _currPartitionIndex = 0; + List _managedPartitions = new ArrayList(); + + public KafkaSpout(SpoutConfig spoutConf, KafkaConfig kafkaConfig) { + _kafkaConfig = kafkaConfig; + _spoutConfig = spoutConf; + } + + @Override + public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + _collector = collector; + Map stateConf = new HashMap(conf); + + List zkServers = _spoutConfig.zkServers; + if(zkServers==null) zkServers = (List) conf.get(Config.STORM_ZOOKEEPER_SERVERS); + + Integer zkPort = _spoutConfig.zkPort; + if(zkPort==null) zkPort = ((Number) conf.get(Config.STORM_ZOOKEEPER_PORT)).intValue(); + + String zkRoot = _spoutConfig.zkRoot; + + stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS, zkServers); + stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_PORT, zkPort); + stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_ROOT, zkRoot); + + // using TransactionalState like this is a hack + _state = TransactionalState.newUserState(stateConf, _spoutConfig.id, null); + _partitions = new KafkaPartitionConnections(_kafkaConfig); + + int totalPartitions = _kafkaConfig.partitionsPerHost * _kafkaConfig.hosts.size(); + int numTasks = context.getComponentTasks(context.getThisComponentId()).size(); + for(int p = context.getThisTaskIndex(); p < totalPartitions; p+=numTasks) { + _managedPartitions.add(p); + _managers.put(p, new PartitionManager(p)); + } + + } + + @Override + public void nextTuple() { + + for(int i=0; i<_managedPartitions.size(); i++) { + int partition = _managedPartitions.get(_currPartitionIndex); + _currPartitionIndex = (_currPartitionIndex + 1) % _managedPartitions.size(); + if(_managers.get(partition).next()) break; + } + + long now = System.currentTimeMillis(); + if((now - _lastUpdateMs) > _spoutConfig.stateUpdateIntervalMs) { + commit(); + } + + } + + @Override + public void ack(Object msgId) { + KafkaMessageId id = (KafkaMessageId) msgId; + _managers.get(id.partition).ack(id.offset); + } + + @Override + public void fail(Object msgId) { + KafkaMessageId id = (KafkaMessageId) msgId; + _managers.get(id.partition).fail(id.offset); + } + + @Override + public void deactivate() { + commit(); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(_kafkaConfig.scheme.getOutputFields()); + } + + private void commit() { + _lastUpdateMs = System.currentTimeMillis(); + for(PartitionManager manager: _managers.values()) { + manager.commit(); + } + } + + + public static void main(String[] args) { + TopologyBuilder builder = new TopologyBuilder(); + List hosts = new ArrayList(); + hosts.add("localhost"); + KafkaConfig kafkaConf = new KafkaConfig(hosts, 3, "nathan"); + kafkaConf.scheme = new StringScheme(); + SpoutConfig spoutConf = new SpoutConfig("/kafka", "id"); + spoutConf.zkServers = new ArrayList() {{ + add("localhost"); + }}; + spoutConf.zkPort = 2181; + + builder.setSpout("spout", + new KafkaSpout(spoutConf, kafkaConf)); + + Config conf = new Config(); + conf.setDebug(true); + + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology("kafka-test", conf, builder.createTopology()); + + Utils.sleep(600000); + } +} diff --git a/src/jvm/storm/kafka/KafkaUtils.java b/src/jvm/storm/kafka/KafkaUtils.java index 72c0a48b87..193d9f3c17 100644 --- a/src/jvm/storm/kafka/KafkaUtils.java +++ b/src/jvm/storm/kafka/KafkaUtils.java @@ -11,7 +11,6 @@ import kafka.javaapi.message.ByteBufferMessageSet; import kafka.message.Message; import kafka.message.MessageAndOffset; -import kafka.message.MessageSet; public class KafkaUtils { diff --git a/src/jvm/storm/kafka/SpoutConfig.java b/src/jvm/storm/kafka/SpoutConfig.java new file mode 100644 index 0000000000..9c4353906d --- /dev/null +++ b/src/jvm/storm/kafka/SpoutConfig.java @@ -0,0 +1,18 @@ +package storm.kafka; + +import java.io.Serializable; +import java.util.List; + + +public class SpoutConfig implements Serializable { + public List zkServers = null; + public Integer zkPort = null; + public String zkRoot = null; + public String id = null; + public long stateUpdateIntervalMs = 2000; + + public SpoutConfig(String zkRoot, String id) { + this.zkRoot = zkRoot; + this.id = id; + } +} From 6c8487da1be2d58bd7addbd2c25418fbd687974e Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 28 Apr 2012 01:39:18 -0700 Subject: [PATCH 071/349] merge spoutconfig with kafkaconfig for regular spout --- src/jvm/storm/kafka/KafkaSpout.java | 23 ++++++++++------------- src/jvm/storm/kafka/SpoutConfig.java | 5 +++-- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index bd81efc378..34454f37df 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -51,7 +51,7 @@ public boolean next() { if(_waitingToEmit.isEmpty()) fill(); MessageAndOffset toEmit = _waitingToEmit.pollFirst(); if(toEmit==null) return false; - List tup = _kafkaConfig.scheme.deserialize(Utils.toByteArray(toEmit.message().payload())); + List tup = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.message().payload())); _collector.emit(tup, new KafkaMessageId(_partition, actualOffset(toEmit))); return true; } @@ -60,10 +60,10 @@ private void fill() { SimpleConsumer consumer = _partitions.getConsumer(_partition); ByteBufferMessageSet msgs = consumer.fetch( new FetchRequest( - _kafkaConfig.topic, + _spoutConfig.topic, _partitions.getHostPartition(_partition), _emittedToOffset, - _kafkaConfig.fetchSizeBytes)); + _spoutConfig.fetchSizeBytes)); for(MessageAndOffset msg: msgs) { _pending.add(actualOffset(msg)); _waitingToEmit.add(msg); @@ -107,7 +107,6 @@ private String committedPath() { } } - KafkaConfig _kafkaConfig; SpoutConfig _spoutConfig; SpoutOutputCollector _collector; TransactionalState _state; @@ -119,8 +118,7 @@ private String committedPath() { int _currPartitionIndex = 0; List _managedPartitions = new ArrayList(); - public KafkaSpout(SpoutConfig spoutConf, KafkaConfig kafkaConfig) { - _kafkaConfig = kafkaConfig; + public KafkaSpout(SpoutConfig spoutConf) { _spoutConfig = spoutConf; } @@ -143,9 +141,9 @@ public void open(Map conf, TopologyContext context, SpoutOutputCollector collect // using TransactionalState like this is a hack _state = TransactionalState.newUserState(stateConf, _spoutConfig.id, null); - _partitions = new KafkaPartitionConnections(_kafkaConfig); + _partitions = new KafkaPartitionConnections(_spoutConfig); - int totalPartitions = _kafkaConfig.partitionsPerHost * _kafkaConfig.hosts.size(); + int totalPartitions = _spoutConfig.partitionsPerHost * _spoutConfig.hosts.size(); int numTasks = context.getComponentTasks(context.getThisComponentId()).size(); for(int p = context.getThisTaskIndex(); p < totalPartitions; p+=numTasks) { _managedPartitions.add(p); @@ -189,7 +187,7 @@ public void deactivate() { @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(_kafkaConfig.scheme.getOutputFields()); + declarer.declare(_spoutConfig.scheme.getOutputFields()); } private void commit() { @@ -204,16 +202,15 @@ public static void main(String[] args) { TopologyBuilder builder = new TopologyBuilder(); List hosts = new ArrayList(); hosts.add("localhost"); - KafkaConfig kafkaConf = new KafkaConfig(hosts, 3, "nathan"); - kafkaConf.scheme = new StringScheme(); - SpoutConfig spoutConf = new SpoutConfig("/kafka", "id"); + SpoutConfig spoutConf = new SpoutConfig(hosts, 3, "nathan", "/kafka", "id"); + spoutConf.scheme = new StringScheme(); spoutConf.zkServers = new ArrayList() {{ add("localhost"); }}; spoutConf.zkPort = 2181; builder.setSpout("spout", - new KafkaSpout(spoutConf, kafkaConf)); + new KafkaSpout(spoutConf)); Config conf = new Config(); conf.setDebug(true); diff --git a/src/jvm/storm/kafka/SpoutConfig.java b/src/jvm/storm/kafka/SpoutConfig.java index 9c4353906d..047161999a 100644 --- a/src/jvm/storm/kafka/SpoutConfig.java +++ b/src/jvm/storm/kafka/SpoutConfig.java @@ -4,14 +4,15 @@ import java.util.List; -public class SpoutConfig implements Serializable { +public class SpoutConfig extends KafkaConfig implements Serializable { public List zkServers = null; public Integer zkPort = null; public String zkRoot = null; public String id = null; public long stateUpdateIntervalMs = 2000; - public SpoutConfig(String zkRoot, String id) { + public SpoutConfig(List hosts, int partitionsPerHost, String topic, String zkRoot, String id) { + super(hosts, partitionsPerHost, topic); this.zkRoot = zkRoot; this.id = id; } From d1b0e48b7b9845b52e15ae6d3e895b279eaa52ba Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 28 Apr 2012 01:45:05 -0700 Subject: [PATCH 072/349] update readme --- README.markdown | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.markdown b/README.markdown index e35c9cb282..504cd46504 100644 --- a/README.markdown +++ b/README.markdown @@ -1,6 +1,4 @@ -storm-kafka provides a TransactionalSpout implementation for Apache Kafka. +storm-kafka provides a regular spout implementation and a TransactionalSpout implementation for Apache Kafka 0.7. -The master branch is compatible with Kafka 0.6, the kafka0.7 branch is compatible with Kafka 0.7. - -storm-kafka is available from Maven [here](http://clojars.org/storm/storm-kafka). A Kafka 0.6 compatible version is under release "0.7-SNAPSHOT", and a Kafka 0.7 compatible version is under release "0.7.0-kafka7-SNAPSHOT". +storm-kafka is available from Maven [here](http://clojars.org/storm/storm-kafka). From e0127153f70caf3afa102a289759f990556465bd Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Tue, 1 May 2012 23:55:53 -0400 Subject: [PATCH 073/349] add ability to start from a particular time, add automatic finding of offsets --- src/jvm/storm/kafka/KafkaConfig.java | 7 +++++ src/jvm/storm/kafka/KafkaSpout.java | 42 ++++++++++++++++++---------- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/jvm/storm/kafka/KafkaConfig.java b/src/jvm/storm/kafka/KafkaConfig.java index 678d1ff0f3..e849c1c4bc 100644 --- a/src/jvm/storm/kafka/KafkaConfig.java +++ b/src/jvm/storm/kafka/KafkaConfig.java @@ -14,10 +14,17 @@ public class KafkaConfig implements Serializable { public int bufferSizeBytes = 1024*1024; public Scheme scheme = new RawScheme(); public String topic; + public long startOffsetTime = -2; + public boolean forceFromStart = false; public KafkaConfig(List hosts, int partitionsPerHost, String topic) { this.hosts = hosts; this.partitionsPerHost = partitionsPerHost; this.topic = topic; } + + public void forceStartOffsetTime(long millis) { + startOffsetTime = millis; + forceFromStart = true; + } } diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index 34454f37df..d2930863e6 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -17,12 +17,16 @@ import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; +import org.apache.log4j.Logger; import kafka.javaapi.message.ByteBufferMessageSet; import kafka.message.MessageAndOffset; import kafka.api.FetchRequest; - +// TODO: need to add blacklisting +// TODO: need to make a best effort to not re-emit messages if don't have to public class KafkaSpout extends BaseRichSpout { + public static final Logger LOG = Logger.getLogger(KafkaSpout.class); + static class KafkaMessageId { public int partition; public long offset; @@ -43,7 +47,12 @@ protected class PartitionManager { public PartitionManager(int partition) { _partition = partition; _committedTo = (Long) _state.getData(committedPath()); - if(_committedTo==null) _committedTo = 0L; + SimpleConsumer consumer = _partitions.getConsumer(_partition); + int hostPartition = _partitions.getHostPartition(_partition); + if(_committedTo==null || _spoutConfig.forceFromStart) { + _committedTo = consumer.getOffsetsBefore(_spoutConfig.topic, hostPartition, _spoutConfig.startOffsetTime, 1)[0]; + } + LOG.info("Starting Kafka " + consumer.host() + ":" + hostPartition + " from offset " + _committedTo); _emittedToOffset = _committedTo; } @@ -58,12 +67,15 @@ public boolean next() { private void fill() { SimpleConsumer consumer = _partitions.getConsumer(_partition); + int hostPartition = _partitions.getHostPartition(_partition); + //LOG.info("Fetching from Kafka: " + consumer.host() + ":" + hostPartition); ByteBufferMessageSet msgs = consumer.fetch( new FetchRequest( _spoutConfig.topic, - _partitions.getHostPartition(_partition), + hostPartition, _emittedToOffset, _spoutConfig.fetchSizeBytes)); + //LOG.info("Fetched " + msgs.underlying().size() + " messages from Kafka: " + consumer.host() + ":" + hostPartition); for(MessageAndOffset msg: msgs) { _pending.add(actualOffset(msg)); _waitingToEmit.add(msg); @@ -153,8 +165,7 @@ public void open(Map conf, TopologyContext context, SpoutOutputCollector collect } @Override - public void nextTuple() { - + public void nextTuple() { for(int i=0; i<_managedPartitions.size(); i++) { int partition = _managedPartitions.get(_currPartitionIndex); _currPartitionIndex = (_currPartitionIndex + 1) % _managedPartitions.size(); @@ -165,7 +176,6 @@ public void nextTuple() { if((now - _lastUpdateMs) > _spoutConfig.stateUpdateIntervalMs) { commit(); } - } @Override @@ -202,18 +212,22 @@ public static void main(String[] args) { TopologyBuilder builder = new TopologyBuilder(); List hosts = new ArrayList(); hosts.add("localhost"); - SpoutConfig spoutConf = new SpoutConfig(hosts, 3, "nathan", "/kafka", "id"); + //hosts.add("smf1-atc-13-sr1.prod.twitter.com"); + //hosts.add("smf1-atx-26-sr1.prod.twitter.com"); + //hosts.add("smf1-atw-05-sr4.prod.twitter.com"); + //hosts.add("smf1-aty-37-sr4.prod.twitter.com"); + SpoutConfig spoutConf = new SpoutConfig(hosts, 8, "clicks", "/kafkastorm", "id"); spoutConf.scheme = new StringScheme(); - spoutConf.zkServers = new ArrayList() {{ - add("localhost"); - }}; - spoutConf.zkPort = 2181; + spoutConf.forceStartOffsetTime(1335774001000L); + // spoutConf.zkServers = new ArrayList() {{ + // add("localhost"); + // }}; + // spoutConf.zkPort = 2181; - builder.setSpout("spout", - new KafkaSpout(spoutConf)); + builder.setSpout("spout", new KafkaSpout(spoutConf), 3); Config conf = new Config(); - conf.setDebug(true); + //conf.setDebug(true); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("kafka-test", conf, builder.createTopology()); From f04ff40d289ce5ef8b172af39a924c0d8ab1609f Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 2 May 2012 15:29:07 -0400 Subject: [PATCH 074/349] round robin only after finishing batch for one partition, fix how resetting of offsets works --- src/jvm/storm/kafka/KafkaSpout.java | 60 +++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index d2930863e6..371e375e68 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -9,6 +9,7 @@ import backtype.storm.topology.base.BaseRichSpout; import backtype.storm.transactional.state.TransactionalState; import backtype.storm.utils.Utils; +import java.io.Serializable; import java.util.ArrayList; import kafka.javaapi.consumer.SimpleConsumer; import java.util.HashMap; @@ -17,6 +18,7 @@ import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; +import java.util.UUID; import org.apache.log4j.Logger; import kafka.javaapi.message.ByteBufferMessageSet; import kafka.message.MessageAndOffset; @@ -25,6 +27,22 @@ // TODO: need to add blacklisting // TODO: need to make a best effort to not re-emit messages if don't have to public class KafkaSpout extends BaseRichSpout { + public static class ZooMeta implements Serializable { + String id; + long offset; + + public ZooMeta(String id, long offset) { + this.id = id; + this.offset = offset; + } + } + + static enum EmitState { + EMITTED_MORE_LEFT, + EMITTED_END, + NO_EMITTED + } + public static final Logger LOG = Logger.getLogger(KafkaSpout.class); static class KafkaMessageId { @@ -46,23 +64,32 @@ protected class PartitionManager { public PartitionManager(int partition) { _partition = partition; - _committedTo = (Long) _state.getData(committedPath()); + ZooMeta zooMeta = (ZooMeta) _state.getData(committedPath()); SimpleConsumer consumer = _partitions.getConsumer(_partition); int hostPartition = _partitions.getHostPartition(_partition); - if(_committedTo==null || _spoutConfig.forceFromStart) { + + //the id stuff makes sure the spout doesn't reset the offset if it restarts + if(zooMeta==null || (!_uuid.equals(zooMeta.id) && _spoutConfig.forceFromStart)) { _committedTo = consumer.getOffsetsBefore(_spoutConfig.topic, hostPartition, _spoutConfig.startOffsetTime, 1)[0]; + } else { + _committedTo = zooMeta.offset; } LOG.info("Starting Kafka " + consumer.host() + ":" + hostPartition + " from offset " + _committedTo); _emittedToOffset = _committedTo; } - public boolean next() { + //returns false if it's reached the end of current batch + public EmitState next() { if(_waitingToEmit.isEmpty()) fill(); MessageAndOffset toEmit = _waitingToEmit.pollFirst(); - if(toEmit==null) return false; + if(toEmit==null) return EmitState.NO_EMITTED; List tup = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.message().payload())); _collector.emit(tup, new KafkaMessageId(_partition, actualOffset(toEmit))); - return true; + if(_waitingToEmit.size()>0) { + return EmitState.EMITTED_MORE_LEFT; + } else { + return EmitState.EMITTED_END; + } } private void fill() { @@ -104,7 +131,7 @@ public void commit() { committedTo = _pending.first(); } if(committedTo!=_committedTo) { - _state.setData(committedPath(), committedTo); + _state.setData(committedPath(), new ZooMeta(_uuid, committedTo)); _committedTo = committedTo; } } @@ -119,6 +146,7 @@ private String committedPath() { } } + String _uuid = UUID.randomUUID().toString(); SpoutConfig _spoutConfig; SpoutOutputCollector _collector; TransactionalState _state; @@ -151,6 +179,9 @@ public void open(Map conf, TopologyContext context, SpoutOutputCollector collect stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_PORT, zkPort); stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_ROOT, zkRoot); + Config componentConf = new Config(); + componentConf.registerSerialization(ZooMeta.class); + // using TransactionalState like this is a hack _state = TransactionalState.newUserState(stateConf, _spoutConfig.id, null); _partitions = new KafkaPartitionConnections(_spoutConfig); @@ -165,11 +196,18 @@ public void open(Map conf, TopologyContext context, SpoutOutputCollector collect } @Override - public void nextTuple() { + public void nextTuple() { + //TODO: change behavior to stick with one partition until it's empty for(int i=0; i<_managedPartitions.size(); i++) { int partition = _managedPartitions.get(_currPartitionIndex); _currPartitionIndex = (_currPartitionIndex + 1) % _managedPartitions.size(); - if(_managers.get(partition).next()) break; + EmitState state = _managers.get(partition).next(); + if(state!=EmitState.EMITTED_MORE_LEFT) { + _currPartitionIndex = (_currPartitionIndex + 1) % _managedPartitions.size(); + } + if(state!=EmitState.NO_EMITTED) { + break; + } } long now = System.currentTimeMillis(); @@ -205,8 +243,7 @@ private void commit() { for(PartitionManager manager: _managers.values()) { manager.commit(); } - } - + } public static void main(String[] args) { TopologyBuilder builder = new TopologyBuilder(); @@ -218,7 +255,8 @@ public static void main(String[] args) { //hosts.add("smf1-aty-37-sr4.prod.twitter.com"); SpoutConfig spoutConf = new SpoutConfig(hosts, 8, "clicks", "/kafkastorm", "id"); spoutConf.scheme = new StringScheme(); - spoutConf.forceStartOffsetTime(1335774001000L); + spoutConf.forceStartOffsetTime(-2); + // spoutConf.zkServers = new ArrayList() {{ // add("localhost"); // }}; From 2730eb89174b9976feb6ca7401eb07832f7b09d4 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 2 May 2012 15:45:09 -0400 Subject: [PATCH 075/349] fix bug --- src/jvm/storm/kafka/KafkaSpout.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index 371e375e68..882588da0c 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -197,10 +197,8 @@ public void open(Map conf, TopologyContext context, SpoutOutputCollector collect @Override public void nextTuple() { - //TODO: change behavior to stick with one partition until it's empty for(int i=0; i<_managedPartitions.size(); i++) { int partition = _managedPartitions.get(_currPartitionIndex); - _currPartitionIndex = (_currPartitionIndex + 1) % _managedPartitions.size(); EmitState state = _managers.get(partition).next(); if(state!=EmitState.EMITTED_MORE_LEFT) { _currPartitionIndex = (_currPartitionIndex + 1) % _managedPartitions.size(); From 8b6fb1dbcf574e975af9368a3660c6d8cd5acc15 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 3 May 2012 01:04:51 -0400 Subject: [PATCH 076/349] formatting --- src/jvm/storm/kafka/KafkaSpout.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index 882588da0c..173bf3d33d 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -247,10 +247,6 @@ public static void main(String[] args) { TopologyBuilder builder = new TopologyBuilder(); List hosts = new ArrayList(); hosts.add("localhost"); - //hosts.add("smf1-atc-13-sr1.prod.twitter.com"); - //hosts.add("smf1-atx-26-sr1.prod.twitter.com"); - //hosts.add("smf1-atw-05-sr4.prod.twitter.com"); - //hosts.add("smf1-aty-37-sr4.prod.twitter.com"); SpoutConfig spoutConf = new SpoutConfig(hosts, 8, "clicks", "/kafkastorm", "id"); spoutConf.scheme = new StringScheme(); spoutConf.forceStartOffsetTime(-2); From 1b519a03ef3230e0ac17c91024e53102bd61bfb1 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 3 May 2012 16:34:51 -0400 Subject: [PATCH 077/349] add noarg constructor to ZooMeta so it works with Kryo --- project.clj | 5 ++++- src/jvm/storm/kafka/KafkaSpout.java | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 889e7c2fba..d5a6239bb5 100644 --- a/project.clj +++ b/project.clj @@ -1,7 +1,10 @@ -(defproject storm/storm-kafka "0.7.1-SNAPSHOT" +(defproject storm/storm-kafka "0.7.1-snap" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} + :repositories {"releases" "http://artifactory.local.twitter.com/libs-releases-local" + "snapshots" "http://artifactory.local.twitter.com/libs-snapshots-local" + "artifactory" "http://artifactory.local.twitter.com/repo"} :dependencies [[storm/kafka "0.7.0-incubating" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index 173bf3d33d..2a81fc3d3d 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -31,6 +31,10 @@ public static class ZooMeta implements Serializable { String id; long offset; + public ZooMeta() { + + } + public ZooMeta(String id, long offset) { this.id = id; this.offset = offset; @@ -183,7 +187,7 @@ public void open(Map conf, TopologyContext context, SpoutOutputCollector collect componentConf.registerSerialization(ZooMeta.class); // using TransactionalState like this is a hack - _state = TransactionalState.newUserState(stateConf, _spoutConfig.id, null); + _state = TransactionalState.newUserState(stateConf, _spoutConfig.id, componentConf); _partitions = new KafkaPartitionConnections(_spoutConfig); int totalPartitions = _spoutConfig.partitionsPerHost * _spoutConfig.hosts.size(); From 8dc6928f6fe808b6219c3d847499655848a8d3a4 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 4 May 2012 10:21:04 -0400 Subject: [PATCH 078/349] update readme --- README.markdown | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/README.markdown b/README.markdown index 504cd46504..12da181aba 100644 --- a/README.markdown +++ b/README.markdown @@ -2,3 +2,51 @@ storm-kafka provides a regular spout implementation and a TransactionalSpout imp storm-kafka is available from Maven [here](http://clojars.org/storm/storm-kafka). +## Using KafkaSpout + +`KafkaSpout` is a regular spout implementation that reads from a Kafka cluster. The basic usage is like this: + +```java +SpoutConfig spoutConfig = new SpoutConfig( + ImmutableList.of("kafkahost1", "kafkahost2"), // list of Kafka brokers + 8, // number of partitions per host + "clicks", // topic to read from + "/kafkastorm", // the root path in Zookeeper for the spout to store the consumer offsets + "discovery"); // an id for this consumer for storing the consumer offsets in Zookeeper +KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig); +``` + +Currently the spout is parameterized with a static list of brokers and a fixed number of partitions per host – we'd like to make this dynamic in the future. + +The spout stores the state of the offsets its consumed in Zookeeper. The spout is parameterized with the root path to store the offsets and an id for this particular spout. So offsets for partitions will be stored in these paths, where "0", "1" are ids for the partitions: + +``` +{root path}/{id}/0 +{root path}/{id}/1 +{root path}/{id}/2 +{root path}/{id}/3 +... +``` + +By default, the offsets will be stored in the same Zookeeper cluster that Storm uses. You can override this via your spout config like this: + +```java +spoutConfig.zkServers = ImmutableList.of("smfd-atm-28-sr3.devel.twitter.com"); +spoutConfig.zkPort = 2191; +``` + +Another very useful config in the spout is the ability to force the spout to rewind to a previous offset. You do `forceStartOffsetTime` on the spout config, like so: + +```java +spoutConfig.forceStartOffsetTime(-2); +``` + +It will choose the latest offset written around that timestamp to start consuming. You can force the spout to always start from the latest offset by passing in -1, and you can force it to start from the earliest offset by passing in -2. + +Finally, `SpoutConfig` has options for adjusting how the spout fetches messages from Kafka (buffer sizes, amount of messages to fetch at a time, timeouts, etc.). + +## TODO + +1. Add blacklisting of Kafka servers in `KafkaSpout` and `OpaqueTransactionalKafkaSpout` +2. Discover Kafka brokers dynamically through Zookeeper instead of using a static list of brokers +3. Discover number of partitions / topic / host dynamically \ No newline at end of file From e258cef74bd9f828df2307acfb5dc7d99e3e75cb Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 4 May 2012 10:23:05 -0400 Subject: [PATCH 079/349] update version --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index d5a6239bb5..1a00eccea4 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.7.1-snap" +(defproject storm/storm-kafka "0.7.1-SNAPSHOT" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} From a1710d77377829826c34ff03f3b4ce6a2a739790 Mon Sep 17 00:00:00 2001 From: Jeroen van Dijk Date: Sat, 5 May 2012 23:30:26 +0300 Subject: [PATCH 080/349] Twitter libraries with version 2.2.5-SNAPSHOT seem to be not available anymore, 2.2.6-SNAPSHOT is. --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index bbc80147b0..f136d6f07a 100644 --- a/project.clj +++ b/project.clj @@ -11,8 +11,8 @@ :dependencies [[org.clojure/clojure "1.2.0"] [org.clojure/clojure-contrib "1.2.0"] - [org.twitter4j/twitter4j-core "2.2.5-SNAPSHOT"] - [org.twitter4j/twitter4j-stream "2.2.5-SNAPSHOT"] + [org.twitter4j/twitter4j-core "2.2.6-SNAPSHOT"] + [org.twitter4j/twitter4j-stream "2.2.6-SNAPSHOT"] ] :dev-dependencies [[storm "0.7.1"] From 4063934ac0027bca38480138f2db56d508c44f7e Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sun, 6 May 2012 22:18:46 -0700 Subject: [PATCH 081/349] update readme --- README.markdown | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.markdown b/README.markdown index 12da181aba..0e7a1327d4 100644 --- a/README.markdown +++ b/README.markdown @@ -31,7 +31,7 @@ The spout stores the state of the offsets its consumed in Zookeeper. The spout i By default, the offsets will be stored in the same Zookeeper cluster that Storm uses. You can override this via your spout config like this: ```java -spoutConfig.zkServers = ImmutableList.of("smfd-atm-28-sr3.devel.twitter.com"); +spoutConfig.zkServers = ImmutableList.of("otherserver.com"); spoutConfig.zkPort = 2191; ``` @@ -49,4 +49,4 @@ Finally, `SpoutConfig` has options for adjusting how the spout fetches messages 1. Add blacklisting of Kafka servers in `KafkaSpout` and `OpaqueTransactionalKafkaSpout` 2. Discover Kafka brokers dynamically through Zookeeper instead of using a static list of brokers -3. Discover number of partitions / topic / host dynamically \ No newline at end of file +3. Discover number of partitions / topic / host dynamically From 818287f347b43ac54741f1e54eb384d952a54e78 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 7 May 2012 10:30:39 -0700 Subject: [PATCH 082/349] update twitter4j deps in m2-pom --- m2-pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/m2-pom.xml b/m2-pom.xml index 192f97869d..b6e9c6d2ed 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -45,12 +45,12 @@ org.twitter4j twitter4j-core - 2.2.5-SNAPSHOT + 2.2.6-SNAPSHOT org.twitter4j twitter4j-stream - 2.2.5-SNAPSHOT + 2.2.6-SNAPSHOT org.clojure From 785ac24971acc4565d987491d239b8e9e6c2f1ca Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 16 May 2012 23:49:03 -0700 Subject: [PATCH 083/349] update storm-kafka to 0.7.2 --- project.clj | 7 +++---- src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java | 6 +++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/project.clj b/project.clj index 1a00eccea4..1327b052af 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.7.1-SNAPSHOT" +(defproject storm/storm-kafka "0.7.2-SNAPSHOT" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} @@ -8,8 +8,7 @@ :dependencies [[storm/kafka "0.7.0-incubating" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.7.1"] - [org.clojure/clojure "1.2.0"] - [org.clojure/clojure-contrib "1.2.0"]] + :dev-dependencies [[storm "0.7.2-rc1"] + [org.clojure/clojure "1.4.0"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) diff --git a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java index f9e08cf726..b7a853415a 100644 --- a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java +++ b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java @@ -53,7 +53,11 @@ class Coordinator implements IOpaquePartitionedTransactionalSpout.Coordinator { public boolean isReady() { //TODO: can do a more sophisticated strategy by looking at the high water marks for each partition return true; - } + } + + @Override + public void close() { + } } class Emitter implements IOpaquePartitionedTransactionalSpout.Emitter { From 24fbd922b60213f776c68b2402e88d22ff50d254 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 25 May 2012 16:14:26 -0700 Subject: [PATCH 084/349] upgrade to 0.7.2-rc1 --- project.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index f136d6f07a..70cf80b443 100644 --- a/project.clj +++ b/project.clj @@ -9,12 +9,12 @@ "twitter4j" "http://twitter4j.org/maven2" } - :dependencies [[org.clojure/clojure "1.2.0"] - [org.clojure/clojure-contrib "1.2.0"] + :dependencies [ [org.twitter4j/twitter4j-core "2.2.6-SNAPSHOT"] [org.twitter4j/twitter4j-stream "2.2.6-SNAPSHOT"] ] - :dev-dependencies [[storm "0.7.1"] + :dev-dependencies [[storm "0.7.2-rc1"] + [org.clojure/clojure "1.4.0"] ]) From fd232215a58bd16707bc8bb93552e7bfbc97914d Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 30 May 2012 17:49:44 -0700 Subject: [PATCH 085/349] bump to 0.7.2-rc2 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 70cf80b443..213f324c0c 100644 --- a/project.clj +++ b/project.clj @@ -14,7 +14,7 @@ [org.twitter4j/twitter4j-stream "2.2.6-SNAPSHOT"] ] - :dev-dependencies [[storm "0.7.2-rc1"] + :dev-dependencies [[storm "0.7.2-rc2"] [org.clojure/clojure "1.4.0"] ]) From 0f73b6a75def2a5cc2adca61e9a9d3c4adbb5868 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 30 May 2012 20:39:21 -0700 Subject: [PATCH 086/349] make kafka spouts more flexible to support kafka hosts on different ports --- src/jvm/storm/kafka/HostPort.java | 15 +++++++++++ src/jvm/storm/kafka/KafkaConfig.java | 26 ++++++++++++++++--- .../kafka/KafkaPartitionConnections.java | 3 ++- src/jvm/storm/kafka/KafkaSpout.java | 2 +- src/jvm/storm/kafka/SpoutConfig.java | 6 ++++- src/jvm/storm/kafka/TestTopology.java | 2 +- 6 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 src/jvm/storm/kafka/HostPort.java diff --git a/src/jvm/storm/kafka/HostPort.java b/src/jvm/storm/kafka/HostPort.java new file mode 100644 index 0000000000..cba7c2237b --- /dev/null +++ b/src/jvm/storm/kafka/HostPort.java @@ -0,0 +1,15 @@ +package storm.kafka; + +public class HostPort { + public String host; + public int port; + + public HostPort(String host, int port) { + this.host = host; + this.port = port; + } + + public HostPort(String host) { + this(host, 9092); + } +} diff --git a/src/jvm/storm/kafka/KafkaConfig.java b/src/jvm/storm/kafka/KafkaConfig.java index e849c1c4bc..a12e4cb8f3 100644 --- a/src/jvm/storm/kafka/KafkaConfig.java +++ b/src/jvm/storm/kafka/KafkaConfig.java @@ -3,11 +3,11 @@ import backtype.storm.spout.RawScheme; import backtype.storm.spout.Scheme; import java.io.Serializable; +import java.util.ArrayList; import java.util.List; public class KafkaConfig implements Serializable { - public List hosts; - public int port = 9092; + public List hosts; public int partitionsPerHost; public int fetchSizeBytes = 1024*1024; public int socketTimeoutMs = 10000; @@ -17,14 +17,34 @@ public class KafkaConfig implements Serializable { public long startOffsetTime = -2; public boolean forceFromStart = false; - public KafkaConfig(List hosts, int partitionsPerHost, String topic) { + public KafkaConfig(List hosts, int partitionsPerHost, String topic) { this.hosts = hosts; this.partitionsPerHost = partitionsPerHost; this.topic = topic; } + public static KafkaConfig fromHostStrings(List hosts, int partitionsPerHost, String topic) { + return new KafkaConfig(convertHosts(hosts), partitionsPerHost, topic); + } + public void forceStartOffsetTime(long millis) { startOffsetTime = millis; forceFromStart = true; } + + public static List convertHosts(List hosts) { + List ret = new ArrayList(); + for(String s: hosts) { + HostPort hp; + String[] spec = s.split(":"); + if(spec.length==1) { + hp = new HostPort(spec[0]); + } else if (spec.length==2) { + hp = new HostPort(spec[0], Integer.parseInt(spec[1])); + } else { + throw new IllegalArgumentException("Invalid host specification: " + s); + } + } + return ret; + } } diff --git a/src/jvm/storm/kafka/KafkaPartitionConnections.java b/src/jvm/storm/kafka/KafkaPartitionConnections.java index 218f399d3f..20d6581721 100644 --- a/src/jvm/storm/kafka/KafkaPartitionConnections.java +++ b/src/jvm/storm/kafka/KafkaPartitionConnections.java @@ -16,7 +16,8 @@ public KafkaPartitionConnections(KafkaConfig conf) { public SimpleConsumer getConsumer(int partition) { int hostIndex = partition / _config.partitionsPerHost; if(!_kafka.containsKey(hostIndex)) { - _kafka.put(hostIndex, new SimpleConsumer(_config.hosts.get(hostIndex), _config.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); + HostPort hp = _config.hosts.get(hostIndex); + _kafka.put(hostIndex, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); } return _kafka.get(hostIndex); diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index 2a81fc3d3d..4bfa0b5a6f 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -251,7 +251,7 @@ public static void main(String[] args) { TopologyBuilder builder = new TopologyBuilder(); List hosts = new ArrayList(); hosts.add("localhost"); - SpoutConfig spoutConf = new SpoutConfig(hosts, 8, "clicks", "/kafkastorm", "id"); + SpoutConfig spoutConf = SpoutConfig.fromHostStrings(hosts, 8, "clicks", "/kafkastorm", "id"); spoutConf.scheme = new StringScheme(); spoutConf.forceStartOffsetTime(-2); diff --git a/src/jvm/storm/kafka/SpoutConfig.java b/src/jvm/storm/kafka/SpoutConfig.java index 047161999a..0b40045bc7 100644 --- a/src/jvm/storm/kafka/SpoutConfig.java +++ b/src/jvm/storm/kafka/SpoutConfig.java @@ -11,9 +11,13 @@ public class SpoutConfig extends KafkaConfig implements Serializable { public String id = null; public long stateUpdateIntervalMs = 2000; - public SpoutConfig(List hosts, int partitionsPerHost, String topic, String zkRoot, String id) { + public SpoutConfig(List hosts, int partitionsPerHost, String topic, String zkRoot, String id) { super(hosts, partitionsPerHost, topic); this.zkRoot = zkRoot; this.id = id; } + + public static SpoutConfig fromHostStrings(List hosts, int partitionsPerHost, String topic, String zkRoot, String id) { + return new SpoutConfig(KafkaConfig.convertHosts(hosts), partitionsPerHost, topic, zkRoot, id); + } } diff --git a/src/jvm/storm/kafka/TestTopology.java b/src/jvm/storm/kafka/TestTopology.java index 01155de2ac..0c0e1644cf 100644 --- a/src/jvm/storm/kafka/TestTopology.java +++ b/src/jvm/storm/kafka/TestTopology.java @@ -26,7 +26,7 @@ public void execute(Tuple tuple, BasicOutputCollector collector) { public static void main(String [] args) throws Exception { List hosts = new ArrayList(); hosts.add("localhost"); - KafkaConfig kafkaConf = new KafkaConfig(hosts, 3, "test"); + KafkaConfig kafkaConf = KafkaConfig.fromHostStrings(hosts, 3, "test"); kafkaConf.scheme = new StringScheme(); LocalCluster cluster = new LocalCluster(); TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("id", "spout", From 6dd85c9549a51bf85d4954e33b023d7241b45882 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 30 May 2012 21:37:44 -0700 Subject: [PATCH 087/349] fix problem with hostport conversion, logging --- src/jvm/storm/kafka/HostPort.java | 4 +++- src/jvm/storm/kafka/KafkaConfig.java | 1 + src/jvm/storm/kafka/KafkaSpout.java | 5 ++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/jvm/storm/kafka/HostPort.java b/src/jvm/storm/kafka/HostPort.java index cba7c2237b..7de26e6555 100644 --- a/src/jvm/storm/kafka/HostPort.java +++ b/src/jvm/storm/kafka/HostPort.java @@ -1,6 +1,8 @@ package storm.kafka; -public class HostPort { +import java.io.Serializable; + +public class HostPort implements Serializable { public String host; public int port; diff --git a/src/jvm/storm/kafka/KafkaConfig.java b/src/jvm/storm/kafka/KafkaConfig.java index a12e4cb8f3..cf0866df5f 100644 --- a/src/jvm/storm/kafka/KafkaConfig.java +++ b/src/jvm/storm/kafka/KafkaConfig.java @@ -44,6 +44,7 @@ public static List convertHosts(List hosts) { } else { throw new IllegalArgumentException("Invalid host specification: " + s); } + ret.add(hp); } return ret; } diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index 4bfa0b5a6f..c530950f1f 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -84,9 +84,12 @@ public PartitionManager(int partition) { //returns false if it's reached the end of current batch public EmitState next() { + LOG.info("Filling from " + _partitions.getConsumer(_partition).host() + " " + _partition); if(_waitingToEmit.isEmpty()) fill(); MessageAndOffset toEmit = _waitingToEmit.pollFirst(); - if(toEmit==null) return EmitState.NO_EMITTED; + if(toEmit==null) { + return EmitState.NO_EMITTED; + } List tup = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.message().payload())); _collector.emit(tup, new KafkaMessageId(_partition, actualOffset(toEmit))); if(_waitingToEmit.size()>0) { From 2654edcd075459178f42808feee994aff9ff6a6d Mon Sep 17 00:00:00 2001 From: Michael Cetrulo Date: Thu, 31 May 2012 23:24:19 -0300 Subject: [PATCH 088/349] show separate instructions for running Java and Clojure examples --- README.markdown | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index c7b1052381..6ea5cc8a31 100644 --- a/README.markdown +++ b/README.markdown @@ -12,10 +12,20 @@ More information about Storm can be found on the [project page](http://github.co Install Leiningen by following the installation instructions [here](https://github.com/technomancy/leiningen). +### To run a Java example: + +``` +lein deps +lein compile +java -cp $(lein classpath) storm.starter.ExclamationTopology +``` + +### To run a Clojure example: + ``` lein deps lein compile -java -cp `lein classpath` storm.starter.ExclamationTopology +lein run -m storm.starter.clj.word-count ``` ## Maven From babc6fd443fa3476bb893951b241ceb836c409ed Mon Sep 17 00:00:00 2001 From: Michael Cetrulo Date: Thu, 31 May 2012 23:40:21 -0300 Subject: [PATCH 089/349] replicate behavior of Main from Java example --- src/clj/storm/starter/clj/word_count.clj | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/clj/storm/starter/clj/word_count.clj b/src/clj/storm/starter/clj/word_count.clj index dc4db280a4..ce2725d88a 100644 --- a/src/clj/storm/starter/clj/word_count.clj +++ b/src/clj/storm/starter/clj/word_count.clj @@ -65,10 +65,16 @@ (.shutdown cluster) )) -(defn -main [name] +(defn submit-topology! [name] (StormSubmitter/submitTopology name {TOPOLOGY-DEBUG true TOPOLOGY-WORKERS 3} (mk-topology))) +(defn -main + ([] + (run-local!)) + ([name] + (submit-topology! name))) + From 72db76e9d761ec345071b4d1c0c6c8315ce1410d Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Tue, 5 Jun 2012 12:19:36 -0700 Subject: [PATCH 090/349] upgrade storm-starter to 0.7.2 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 213f324c0c..700bdb6cbd 100644 --- a/project.clj +++ b/project.clj @@ -14,7 +14,7 @@ [org.twitter4j/twitter4j-stream "2.2.6-SNAPSHOT"] ] - :dev-dependencies [[storm "0.7.2-rc2"] + :dev-dependencies [[storm "0.7.2"] [org.clojure/clojure "1.4.0"] ]) From d68498e3d4afb153807140e2f1160a29949fd0b8 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 11 Jun 2012 00:00:04 -0700 Subject: [PATCH 091/349] update pom for 0.7.2 --- m2-pom.xml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/m2-pom.xml b/m2-pom.xml index b6e9c6d2ed..b7e34812f3 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -38,7 +38,7 @@ storm storm - 0.7.1 + 0.7.2 provided @@ -52,16 +52,6 @@ twitter4j-stream 2.2.6-SNAPSHOT - - org.clojure - clojure - 1.2.0 - - - org.clojure - clojure-contrib - 1.2.0 - com.googlecode.json-simple json-simple From 2d9db4420dd745ee9a19b2dbc04ee1f97d29ca3f Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 11 Jun 2012 17:33:18 -0700 Subject: [PATCH 092/349] use prior message's next offset for current message's offset rather than trying to compute it manually --- src/jvm/storm/kafka/KafkaSpout.java | 35 ++++++++------- src/jvm/storm/kafka/TestTwitterTopology.java | 45 ++++++++++++++++++++ 2 files changed, 65 insertions(+), 15 deletions(-) create mode 100644 src/jvm/storm/kafka/TestTwitterTopology.java diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index c530950f1f..d54f89630b 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -23,6 +23,7 @@ import kafka.javaapi.message.ByteBufferMessageSet; import kafka.message.MessageAndOffset; import kafka.api.FetchRequest; +import kafka.message.Message; // TODO: need to add blacklisting // TODO: need to make a best effort to not re-emit messages if don't have to @@ -41,6 +42,16 @@ public ZooMeta(String id, long offset) { } } + public static class MessageAndRealOffset { + public Message msg; + public long offset; + + public MessageAndRealOffset(Message msg, long offset) { + this.msg = msg; + this.offset = offset; + } + } + static enum EmitState { EMITTED_MORE_LEFT, EMITTED_END, @@ -64,7 +75,7 @@ protected class PartitionManager { SortedSet _pending = new TreeSet(); Long _committedTo; int _partition; - LinkedList _waitingToEmit = new LinkedList(); + LinkedList _waitingToEmit = new LinkedList(); public PartitionManager(int partition) { _partition = partition; @@ -84,14 +95,13 @@ public PartitionManager(int partition) { //returns false if it's reached the end of current batch public EmitState next() { - LOG.info("Filling from " + _partitions.getConsumer(_partition).host() + " " + _partition); if(_waitingToEmit.isEmpty()) fill(); - MessageAndOffset toEmit = _waitingToEmit.pollFirst(); + MessageAndRealOffset toEmit = _waitingToEmit.pollFirst(); if(toEmit==null) { return EmitState.NO_EMITTED; } - List tup = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.message().payload())); - _collector.emit(tup, new KafkaMessageId(_partition, actualOffset(toEmit))); + List tup = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.msg.payload())); + _collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset)); if(_waitingToEmit.size()>0) { return EmitState.EMITTED_MORE_LEFT; } else { @@ -102,17 +112,17 @@ public EmitState next() { private void fill() { SimpleConsumer consumer = _partitions.getConsumer(_partition); int hostPartition = _partitions.getHostPartition(_partition); - //LOG.info("Fetching from Kafka: " + consumer.host() + ":" + hostPartition); + LOG.info("Fetching from Kafka: " + consumer.host() + ":" + hostPartition + " from offset " + _emittedToOffset); ByteBufferMessageSet msgs = consumer.fetch( new FetchRequest( _spoutConfig.topic, hostPartition, _emittedToOffset, _spoutConfig.fetchSizeBytes)); - //LOG.info("Fetched " + msgs.underlying().size() + " messages from Kafka: " + consumer.host() + ":" + hostPartition); + LOG.info("Fetched " + msgs.underlying().size() + " messages from Kafka: " + consumer.host() + ":" + hostPartition); for(MessageAndOffset msg: msgs) { - _pending.add(actualOffset(msg)); - _waitingToEmit.add(msg); + _pending.add(_emittedToOffset); + _waitingToEmit.add(new MessageAndRealOffset(msg.message(), _emittedToOffset)); _emittedToOffset = msg.offset(); } } @@ -142,12 +152,7 @@ public void commit() { _committedTo = committedTo; } } - - private long actualOffset(MessageAndOffset msg) { - return msg.offset() - msg.message().serializedSize(); - - } - + private String committedPath() { return _spoutConfig.id + "/" + _partition; } diff --git a/src/jvm/storm/kafka/TestTwitterTopology.java b/src/jvm/storm/kafka/TestTwitterTopology.java new file mode 100644 index 0000000000..732bc6c11b --- /dev/null +++ b/src/jvm/storm/kafka/TestTwitterTopology.java @@ -0,0 +1,45 @@ +package storm.kafka; + +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.topology.BasicOutputCollector; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseBasicBolt; +import backtype.storm.transactional.TransactionalTopologyBuilder; +import backtype.storm.tuple.Tuple; +import backtype.storm.StormSubmitter; +import java.util.ArrayList; +import java.util.List; + +public class TestTwitterTopology { + public static class PrinterBolt extends BaseBasicBolt { + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + System.out.println(tuple.toString()); + } + + } + + public static void main(String [] args) throws Exception { + List hosts = new ArrayList(); + hosts.add("smf1-atc-13-sr1.prod.twitter.com"); + hosts.add("smf1-atx-26-sr1.prod.twitter.com"); + hosts.add("smf1-atw-05-sr4.prod.twitter.com"); + hosts.add("smf1-aty-37-sr4.prod.twitter.com"); + KafkaConfig kafkaConf = KafkaConfig.fromHostStrings(hosts, 8, "firehose"); + kafkaConf.scheme = new StringScheme(); + kafkaConf.forceStartOffsetTime(-1); + TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("id", "spout", + new OpaqueTransactionalKafkaSpout(kafkaConf), 1); + builder.setBolt("printer", new PrinterBolt()) + .shuffleGrouping("spout"); + Config config = new Config(); + + StormSubmitter.submitTopology("kafka-test", config, builder.buildTopology()); + + } +} From 880c2940f0a812d0efdb67e34bdaf66d9376452d Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 11 Jun 2012 17:33:28 -0700 Subject: [PATCH 093/349] use 0.7.2 dep --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 1327b052af..379f65ca81 100644 --- a/project.clj +++ b/project.clj @@ -8,7 +8,7 @@ :dependencies [[storm/kafka "0.7.0-incubating" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.7.2-rc1"] + :dev-dependencies [[storm "0.7.2"] [org.clojure/clojure "1.4.0"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) From 654a4365676f61ac25f35297cdd3c2e2411b67bd Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 11 Jun 2012 18:24:38 -0700 Subject: [PATCH 094/349] remove test code --- src/jvm/storm/kafka/TestTwitterTopology.java | 45 -------------------- 1 file changed, 45 deletions(-) delete mode 100644 src/jvm/storm/kafka/TestTwitterTopology.java diff --git a/src/jvm/storm/kafka/TestTwitterTopology.java b/src/jvm/storm/kafka/TestTwitterTopology.java deleted file mode 100644 index 732bc6c11b..0000000000 --- a/src/jvm/storm/kafka/TestTwitterTopology.java +++ /dev/null @@ -1,45 +0,0 @@ -package storm.kafka; - -import backtype.storm.Config; -import backtype.storm.LocalCluster; -import backtype.storm.topology.BasicOutputCollector; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseBasicBolt; -import backtype.storm.transactional.TransactionalTopologyBuilder; -import backtype.storm.tuple.Tuple; -import backtype.storm.StormSubmitter; -import java.util.ArrayList; -import java.util.List; - -public class TestTwitterTopology { - public static class PrinterBolt extends BaseBasicBolt { - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - } - - @Override - public void execute(Tuple tuple, BasicOutputCollector collector) { - System.out.println(tuple.toString()); - } - - } - - public static void main(String [] args) throws Exception { - List hosts = new ArrayList(); - hosts.add("smf1-atc-13-sr1.prod.twitter.com"); - hosts.add("smf1-atx-26-sr1.prod.twitter.com"); - hosts.add("smf1-atw-05-sr4.prod.twitter.com"); - hosts.add("smf1-aty-37-sr4.prod.twitter.com"); - KafkaConfig kafkaConf = KafkaConfig.fromHostStrings(hosts, 8, "firehose"); - kafkaConf.scheme = new StringScheme(); - kafkaConf.forceStartOffsetTime(-1); - TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("id", "spout", - new OpaqueTransactionalKafkaSpout(kafkaConf), 1); - builder.setBolt("printer", new PrinterBolt()) - .shuffleGrouping("spout"); - Config config = new Config(); - - StormSubmitter.submitTopology("kafka-test", config, builder.buildTopology()); - - } -} From f55992d679c844e2929cbe23401cb4d3de1e21ca Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Tue, 12 Jun 2012 17:53:28 -0700 Subject: [PATCH 095/349] bump version --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 379f65ca81..27a8e9ae8c 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.7.2-SNAPSHOT" +(defproject storm/storm-kafka "0.7.3-SNAPSHOT" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} From 41b2b77c044f697919084f0e5c07a8746006f069 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 15 Jun 2012 12:11:32 -0700 Subject: [PATCH 096/349] upgrade to 0.7.3 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 700bdb6cbd..c7698c5f47 100644 --- a/project.clj +++ b/project.clj @@ -14,7 +14,7 @@ [org.twitter4j/twitter4j-stream "2.2.6-SNAPSHOT"] ] - :dev-dependencies [[storm "0.7.2"] + :dev-dependencies [[storm "0.7.3"] [org.clojure/clojure "1.4.0"] ]) From c928c757f9db3a28c3ad6dd9cc40878add6f2c35 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 27 Jun 2012 15:32:24 -0700 Subject: [PATCH 097/349] bump version --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index c7698c5f47..98b7f33d51 100644 --- a/project.clj +++ b/project.clj @@ -14,7 +14,7 @@ [org.twitter4j/twitter4j-stream "2.2.6-SNAPSHOT"] ] - :dev-dependencies [[storm "0.7.3"] + :dev-dependencies [[storm "0.7.4"] [org.clojure/clojure "1.4.0"] ]) From 5869d51f8a77099b309bca07b7a7c4eee46ef79c Mon Sep 17 00:00:00 2001 From: Argyris Zymnis Date: Wed, 27 Jun 2012 17:02:26 -0700 Subject: [PATCH 098/349] Add partitions that are picked up by zookeeper. --- src/jvm/storm/kafka/KafkaConfig.java | 8 +- .../kafka/KafkaPartitionConnections.java | 13 ++- src/jvm/storm/kafka/KafkaSpout.java | 92 ++++++++++--------- src/jvm/storm/kafka/SpoutConfig.java | 11 ++- .../kafka/ZkKafkaPartitionConnections.java | 91 ++++++++++++++++++ 5 files changed, 160 insertions(+), 55 deletions(-) create mode 100644 src/jvm/storm/kafka/ZkKafkaPartitionConnections.java diff --git a/src/jvm/storm/kafka/KafkaConfig.java b/src/jvm/storm/kafka/KafkaConfig.java index cf0866df5f..b2a74d5ef4 100644 --- a/src/jvm/storm/kafka/KafkaConfig.java +++ b/src/jvm/storm/kafka/KafkaConfig.java @@ -16,22 +16,24 @@ public class KafkaConfig implements Serializable { public String topic; public long startOffsetTime = -2; public boolean forceFromStart = false; + public String brokerZkHost = null; + public String brokerZkPath = null; public KafkaConfig(List hosts, int partitionsPerHost, String topic) { this.hosts = hosts; this.partitionsPerHost = partitionsPerHost; this.topic = topic; } - + public static KafkaConfig fromHostStrings(List hosts, int partitionsPerHost, String topic) { return new KafkaConfig(convertHosts(hosts), partitionsPerHost, topic); } - + public void forceStartOffsetTime(long millis) { startOffsetTime = millis; forceFromStart = true; } - + public static List convertHosts(List hosts) { List ret = new ArrayList(); for(String s: hosts) { diff --git a/src/jvm/storm/kafka/KafkaPartitionConnections.java b/src/jvm/storm/kafka/KafkaPartitionConnections.java index 20d6581721..4d8985ee38 100644 --- a/src/jvm/storm/kafka/KafkaPartitionConnections.java +++ b/src/jvm/storm/kafka/KafkaPartitionConnections.java @@ -6,30 +6,29 @@ public class KafkaPartitionConnections { Map _kafka = new HashMap(); - KafkaConfig _config; - + public KafkaPartitionConnections(KafkaConfig conf) { _config = conf; } - + public SimpleConsumer getConsumer(int partition) { int hostIndex = partition / _config.partitionsPerHost; if(!_kafka.containsKey(hostIndex)) { HostPort hp = _config.hosts.get(hostIndex); _kafka.put(hostIndex, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); - + } return _kafka.get(hostIndex); } - + public int getHostPartition(int globalPartition) { return globalPartition % _config.partitionsPerHost; } - + public void close() { for(SimpleConsumer consumer: _kafka.values()) { consumer.close(); } - } + } } diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index d54f89630b..e6dc9c812e 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -31,58 +31,58 @@ public class KafkaSpout extends BaseRichSpout { public static class ZooMeta implements Serializable { String id; long offset; - + public ZooMeta() { - + } - + public ZooMeta(String id, long offset) { this.id = id; this.offset = offset; } } - + public static class MessageAndRealOffset { public Message msg; public long offset; - + public MessageAndRealOffset(Message msg, long offset) { this.msg = msg; this.offset = offset; } } - + static enum EmitState { EMITTED_MORE_LEFT, EMITTED_END, NO_EMITTED } - + public static final Logger LOG = Logger.getLogger(KafkaSpout.class); - + static class KafkaMessageId { public int partition; public long offset; - + public KafkaMessageId(int partition, long offset) { this.partition = partition; this.offset = offset; } } - + protected class PartitionManager { Long _emittedToOffset; SortedSet _pending = new TreeSet(); Long _committedTo; int _partition; LinkedList _waitingToEmit = new LinkedList(); - + public PartitionManager(int partition) { _partition = partition; ZooMeta zooMeta = (ZooMeta) _state.getData(committedPath()); SimpleConsumer consumer = _partitions.getConsumer(_partition); int hostPartition = _partitions.getHostPartition(_partition); - + //the id stuff makes sure the spout doesn't reset the offset if it restarts if(zooMeta==null || (!_uuid.equals(zooMeta.id) && _spoutConfig.forceFromStart)) { _committedTo = consumer.getOffsetsBefore(_spoutConfig.topic, hostPartition, _spoutConfig.startOffsetTime, 1)[0]; @@ -90,9 +90,9 @@ public PartitionManager(int partition) { _committedTo = zooMeta.offset; } LOG.info("Starting Kafka " + consumer.host() + ":" + hostPartition + " from offset " + _committedTo); - _emittedToOffset = _committedTo; + _emittedToOffset = _committedTo; } - + //returns false if it's reached the end of current batch public EmitState next() { if(_waitingToEmit.isEmpty()) fill(); @@ -108,7 +108,7 @@ public EmitState next() { return EmitState.EMITTED_END; } } - + private void fill() { SimpleConsumer consumer = _partitions.getConsumer(_partition); int hostPartition = _partitions.getHostPartition(_partition); @@ -126,11 +126,11 @@ private void fill() { _emittedToOffset = msg.offset(); } } - + public void ack(Long offset) { _pending.remove(offset); } - + public void fail(Long offset) { //TODO: should it use in-memory ack set to skip anything that's been acked but not committed??? // things might get crazy with lots of timeouts @@ -139,7 +139,7 @@ public void fail(Long offset) { _pending.tailSet(offset).clear(); } } - + public void commit() { long committedTo; if(_pending.isEmpty()) { @@ -152,12 +152,12 @@ public void commit() { _committedTo = committedTo; } } - + private String committedPath() { return _spoutConfig.id + "/" + _partition; } } - + String _uuid = UUID.randomUUID().toString(); SpoutConfig _spoutConfig; SpoutOutputCollector _collector; @@ -166,37 +166,43 @@ private String committedPath() { Map _managers = new HashMap(); long _lastUpdateMs = 0; - + int _currPartitionIndex = 0; List _managedPartitions = new ArrayList(); - + public KafkaSpout(SpoutConfig spoutConf) { _spoutConfig = spoutConf; } - + @Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { _collector = collector; Map stateConf = new HashMap(conf); - + List zkServers = _spoutConfig.zkServers; if(zkServers==null) zkServers = (List) conf.get(Config.STORM_ZOOKEEPER_SERVERS); Integer zkPort = _spoutConfig.zkPort; if(zkPort==null) zkPort = ((Number) conf.get(Config.STORM_ZOOKEEPER_PORT)).intValue(); - + String zkRoot = _spoutConfig.zkRoot; - + stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS, zkServers); stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_PORT, zkPort); stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_ROOT, zkRoot); - + Config componentConf = new Config(); componentConf.registerSerialization(ZooMeta.class); - + // using TransactionalState like this is a hack _state = TransactionalState.newUserState(stateConf, _spoutConfig.id, componentConf); - _partitions = new KafkaPartitionConnections(_spoutConfig); + if(_spoutConfig.hosts == null) { + // In this case get hosts fromZK + _partitions = new ZkKafkaPartitionConnections(_spoutConfig); + } else { + // Else just use regular partitions from host list + _partitions = new KafkaPartitionConnections(_spoutConfig); + } int totalPartitions = _spoutConfig.partitionsPerHost * _spoutConfig.hosts.size(); int numTasks = context.getComponentTasks(context.getThisComponentId()).size(); @@ -213,13 +219,13 @@ public void nextTuple() { int partition = _managedPartitions.get(_currPartitionIndex); EmitState state = _managers.get(partition).next(); if(state!=EmitState.EMITTED_MORE_LEFT) { - _currPartitionIndex = (_currPartitionIndex + 1) % _managedPartitions.size(); + _currPartitionIndex = (_currPartitionIndex + 1) % _managedPartitions.size(); } if(state!=EmitState.NO_EMITTED) { break; } } - + long now = System.currentTimeMillis(); if((now - _lastUpdateMs) > _spoutConfig.stateUpdateIntervalMs) { commit(); @@ -241,20 +247,20 @@ public void fail(Object msgId) { @Override public void deactivate() { commit(); - } - + } + @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(_spoutConfig.scheme.getOutputFields()); } - + private void commit() { _lastUpdateMs = System.currentTimeMillis(); for(PartitionManager manager: _managers.values()) { manager.commit(); - } - } - + } + } + public static void main(String[] args) { TopologyBuilder builder = new TopologyBuilder(); List hosts = new ArrayList(); @@ -262,20 +268,20 @@ public static void main(String[] args) { SpoutConfig spoutConf = SpoutConfig.fromHostStrings(hosts, 8, "clicks", "/kafkastorm", "id"); spoutConf.scheme = new StringScheme(); spoutConf.forceStartOffsetTime(-2); - + // spoutConf.zkServers = new ArrayList() {{ - // add("localhost"); + // add("localhost"); // }}; // spoutConf.zkPort = 2181; - + builder.setSpout("spout", new KafkaSpout(spoutConf), 3); - + Config conf = new Config(); //conf.setDebug(true); - + LocalCluster cluster = new LocalCluster(); cluster.submitTopology("kafka-test", conf, builder.createTopology()); - + Utils.sleep(600000); } } diff --git a/src/jvm/storm/kafka/SpoutConfig.java b/src/jvm/storm/kafka/SpoutConfig.java index 0b40045bc7..fbce47fbab 100644 --- a/src/jvm/storm/kafka/SpoutConfig.java +++ b/src/jvm/storm/kafka/SpoutConfig.java @@ -10,14 +10,21 @@ public class SpoutConfig extends KafkaConfig implements Serializable { public String zkRoot = null; public String id = null; public long stateUpdateIntervalMs = 2000; - + public SpoutConfig(List hosts, int partitionsPerHost, String topic, String zkRoot, String id) { super(hosts, partitionsPerHost, topic); this.zkRoot = zkRoot; this.id = id; } - + public static SpoutConfig fromHostStrings(List hosts, int partitionsPerHost, String topic, String zkRoot, String id) { return new SpoutConfig(KafkaConfig.convertHosts(hosts), partitionsPerHost, topic, zkRoot, id); } + + public static SpoutConfig fromZkServer(String brokerZkHost, String brokerZkPath, int partitionsPerHost, String topic, String zkRoot, String id) { + SpoutConfig conf = new SpoutConfig(null, partitionsPerHost, topic, zkRoot, id); + conf.brokerZkHost = brokerZkHost; + conf.brokerZkPath = brokerZkPath; + return conf; + } } diff --git a/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java b/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java new file mode 100644 index 0000000000..8f9c79b306 --- /dev/null +++ b/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java @@ -0,0 +1,91 @@ +package storm.kafka; + +import java.io.IOException; +import java.lang.InterruptedException; +import java.lang.RuntimeException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import kafka.javaapi.consumer.SimpleConsumer; + +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.Watcher.Event.EventType; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.Stat; + +public class ZkKafkaPartitionConnections extends KafkaPartitionConnections { + private static int ZK_TIMEOUT = 10000; + + Map _pathToHostIndex = new HashMap(); + ZooKeeper _zkClient; + String _brokerZkPath; + + // TODO: Only handles node data changes. + // Should we handle the case where new producers enter or + // old ones leave? + Watcher eventWatcher = new Watcher() { + @Override + public void process(WatchedEvent event) { + if(event.getType() == EventType.NodeDataChanged) { + String path = event.getPath(); + HostPort hp = getBrokerHost(path); + int hostIndex = _pathToHostIndex.get(path); + _kafka.put(hostIndex, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); + } + } + }; + + public ZkKafkaPartitionConnections(KafkaConfig conf) { + super(conf); + _brokerZkPath = conf.brokerZkPath; + try { + _zkClient = new ZooKeeper(conf.brokerZkHost, 2000, eventWatcher); + } catch(IOException e) { + // Screw you checked exceptions! + throw new RuntimeException("Error while creating zkClient: " + e); + } + setupHosts(); + } + + @Override + public SimpleConsumer getConsumer(int partition) { + int hostIndex = partition / _config.partitionsPerHost; + if(!_kafka.containsKey(hostIndex)) { + HostPort hp = _config.hosts.get(hostIndex); + _kafka.put(hostIndex, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); + } + return _kafka.get(hostIndex); + } + + private HostPort getBrokerHost(String zkPath) { + try { + String zkData = new String(_zkClient.getData(_brokerZkPath + "/" + zkPath, true, new Stat())); + String[] hostString = zkData.split(":"); + String host = hostString[hostString.length - 1]; + int port = Integer.parseInt(hostString[hostString.length - 2]); + return new HostPort(host, port); + } catch(KeeperException e) { + throw new RuntimeException("Could not get broker data: " + e); + } catch(InterruptedException e) { + throw new RuntimeException("Error while creating zkClient: " + e); + } + } + + private void setupHosts() { + try { + List brokers = _zkClient.getChildren(_brokerZkPath, false); + for(int i = 0; i < brokers.size(); i++) { + String path = brokers.get(i); + HostPort hp = getBrokerHost(path); + _kafka.put(i, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); + } + } catch(KeeperException e) { + throw new RuntimeException("Could not get brokers: " + e); + } catch(InterruptedException e) { + throw new RuntimeException("Error while creating zkClient: " + e); + } + } +} \ No newline at end of file From 0e1eb41a5988a781abfc1ef397c0708090befd21 Mon Sep 17 00:00:00 2001 From: Argyris Zymnis Date: Thu, 28 Jun 2012 10:41:12 -0700 Subject: [PATCH 099/349] Also deal with number of hosts in partition connections class. --- .../storm/kafka/KafkaPartitionConnections.java | 4 ++++ src/jvm/storm/kafka/KafkaSpout.java | 2 +- .../kafka/ZkKafkaPartitionConnections.java | 17 +++++++++++++---- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/jvm/storm/kafka/KafkaPartitionConnections.java b/src/jvm/storm/kafka/KafkaPartitionConnections.java index 4d8985ee38..ce9835102e 100644 --- a/src/jvm/storm/kafka/KafkaPartitionConnections.java +++ b/src/jvm/storm/kafka/KafkaPartitionConnections.java @@ -26,6 +26,10 @@ public int getHostPartition(int globalPartition) { return globalPartition % _config.partitionsPerHost; } + public int getNumberOfHosts() { + return _config.hosts.size(); + } + public void close() { for(SimpleConsumer consumer: _kafka.values()) { consumer.close(); diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index e6dc9c812e..c84f9d9d6c 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -204,7 +204,7 @@ public void open(Map conf, TopologyContext context, SpoutOutputCollector collect _partitions = new KafkaPartitionConnections(_spoutConfig); } - int totalPartitions = _spoutConfig.partitionsPerHost * _spoutConfig.hosts.size(); + int totalPartitions = _spoutConfig.partitionsPerHost * _partitions.getNumberOfHosts(); int numTasks = context.getComponentTasks(context.getThisComponentId()).size(); for(int p = context.getThisTaskIndex(); p < totalPartitions; p+=numTasks) { _managedPartitions.add(p); diff --git a/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java b/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java index 8f9c79b306..8d6c956d42 100644 --- a/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java +++ b/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java @@ -34,6 +34,8 @@ public void process(WatchedEvent event) { HostPort hp = getBrokerHost(path); int hostIndex = _pathToHostIndex.get(path); _kafka.put(hostIndex, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); + } else { + throw new RuntimeException("Kafka producer node changed but don't know how to deal with event: " + event.getType()); } } }; @@ -50,16 +52,23 @@ public ZkKafkaPartitionConnections(KafkaConfig conf) { setupHosts(); } + @Override + public int getNumberOfHosts() { + return _kafka.size(); + } + @Override public SimpleConsumer getConsumer(int partition) { int hostIndex = partition / _config.partitionsPerHost; - if(!_kafka.containsKey(hostIndex)) { - HostPort hp = _config.hosts.get(hostIndex); - _kafka.put(hostIndex, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); - } return _kafka.get(hostIndex); } + @Override + public void close() { + super.close(); + _zkClient.close(); + } + private HostPort getBrokerHost(String zkPath) { try { String zkData = new String(_zkClient.getData(_brokerZkPath + "/" + zkPath, true, new Stat())); From d4488c80c1aaacd685484d27bf8f5b91b77f8d33 Mon Sep 17 00:00:00 2001 From: Argyris Zymnis Date: Thu, 28 Jun 2012 10:45:57 -0700 Subject: [PATCH 100/349] Add the newline. --- src/jvm/storm/kafka/ZkKafkaPartitionConnections.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java b/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java index 8d6c956d42..1cf9550763 100644 --- a/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java +++ b/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java @@ -97,4 +97,4 @@ private void setupHosts() { throw new RuntimeException("Error while creating zkClient: " + e); } } -} \ No newline at end of file +} From 4a12919b66ad45ac86789c6c74300cb3e0c31d10 Mon Sep 17 00:00:00 2001 From: Argyris Zymnis Date: Thu, 28 Jun 2012 15:56:44 -0700 Subject: [PATCH 101/349] A couple more fixes and tested to make sure that this works. --- src/jvm/storm/kafka/ZkKafkaPartitionConnections.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java b/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java index 1cf9550763..c5d3491d26 100644 --- a/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java +++ b/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java @@ -66,15 +66,19 @@ public SimpleConsumer getConsumer(int partition) { @Override public void close() { super.close(); - _zkClient.close(); + try { + _zkClient.close(); + } catch(InterruptedException e) { + throw new RuntimeException("Error while closing client"); + } } private HostPort getBrokerHost(String zkPath) { try { String zkData = new String(_zkClient.getData(_brokerZkPath + "/" + zkPath, true, new Stat())); String[] hostString = zkData.split(":"); - String host = hostString[hostString.length - 1]; - int port = Integer.parseInt(hostString[hostString.length - 2]); + String host = hostString[hostString.length - 2]; + int port = Integer.parseInt(hostString[hostString.length - 1]); return new HostPort(host, port); } catch(KeeperException e) { throw new RuntimeException("Could not get broker data: " + e); From d745a9435dd698ba3eaa8cb8db44f864b87ddd90 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 2 Jul 2012 18:38:52 -0700 Subject: [PATCH 102/349] implement dynamic hosts / partitionsPerHost for KafkaSpout --- .../kafka/DynamicPartitionConnections.java | 51 +++++ src/jvm/storm/kafka/GlobalPartitionId.java | 25 +++ src/jvm/storm/kafka/HostPort.java | 11 + src/jvm/storm/kafka/KafkaConfig.java | 49 ++++- src/jvm/storm/kafka/KafkaSpout.java | 199 +++++------------- src/jvm/storm/kafka/KafkaUtils.java | 4 +- .../kafka/OpaqueTransactionalKafkaSpout.java | 8 +- src/jvm/storm/kafka/PartitionCoordinator.java | 8 + src/jvm/storm/kafka/PartitionManager.java | 126 +++++++++++ src/jvm/storm/kafka/SpoutConfig.java | 15 +- src/jvm/storm/kafka/StaticCoordinator.java | 41 ++++ ...s.java => StaticPartitionConnections.java} | 21 +- src/jvm/storm/kafka/TestTopology.java | 3 +- .../storm/kafka/TransactionalKafkaSpout.java | 13 +- src/jvm/storm/kafka/ZkCoordinator.java | 147 +++++++++++++ .../kafka/ZkKafkaPartitionConnections.java | 104 --------- 16 files changed, 531 insertions(+), 294 deletions(-) create mode 100644 src/jvm/storm/kafka/DynamicPartitionConnections.java create mode 100644 src/jvm/storm/kafka/GlobalPartitionId.java create mode 100644 src/jvm/storm/kafka/PartitionCoordinator.java create mode 100644 src/jvm/storm/kafka/PartitionManager.java create mode 100644 src/jvm/storm/kafka/StaticCoordinator.java rename src/jvm/storm/kafka/{KafkaPartitionConnections.java => StaticPartitionConnections.java} (56%) create mode 100644 src/jvm/storm/kafka/ZkCoordinator.java delete mode 100644 src/jvm/storm/kafka/ZkKafkaPartitionConnections.java diff --git a/src/jvm/storm/kafka/DynamicPartitionConnections.java b/src/jvm/storm/kafka/DynamicPartitionConnections.java new file mode 100644 index 0000000000..adf7b5aed1 --- /dev/null +++ b/src/jvm/storm/kafka/DynamicPartitionConnections.java @@ -0,0 +1,51 @@ +package storm.kafka; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import kafka.javaapi.consumer.SimpleConsumer; + + +public class DynamicPartitionConnections { + static class ConnectionInfo { + SimpleConsumer consumer; + Set partitions = new HashSet(); + + public ConnectionInfo(SimpleConsumer consumer) { + this.consumer = consumer; + } + } + + Map _connections = new HashMap(); + SpoutConfig _config; + + public DynamicPartitionConnections(SpoutConfig config) { + _config = config; + } + + public SimpleConsumer register(HostPort host, int partition) { + if(!_connections.containsKey(host)) { + _connections.put(host, new ConnectionInfo(new SimpleConsumer(host.host, host.port, _config.socketTimeoutMs, _config.bufferSizeBytes))); + } + ConnectionInfo info = _connections.get(host); + info.partitions.add(partition); + return info.consumer; + } + + public void unregister(HostPort port, int partition) { + ConnectionInfo info = _connections.get(port); + info.partitions.remove(partition); + if(info.partitions.size()==0) { + info.consumer.close(); + } + _connections.remove(port); + } + + public void close() { + for(ConnectionInfo info: _connections.values()) { + info.consumer.close(); + } + _connections.clear(); + } +} diff --git a/src/jvm/storm/kafka/GlobalPartitionId.java b/src/jvm/storm/kafka/GlobalPartitionId.java new file mode 100644 index 0000000000..bbe04cf631 --- /dev/null +++ b/src/jvm/storm/kafka/GlobalPartitionId.java @@ -0,0 +1,25 @@ +package storm.kafka; + + +public class GlobalPartitionId { + HostPort host; + int partition; + + public GlobalPartitionId(HostPort host, int partition) { + this.host = host; + this.partition = partition; + } + + @Override + public boolean equals(Object o) { + GlobalPartitionId other = (GlobalPartitionId) o; + return host.equals(other.host) && partition == other.partition; + } + + @Override + public int hashCode() { + return 13 * host.hashCode() + partition; + } + + +} diff --git a/src/jvm/storm/kafka/HostPort.java b/src/jvm/storm/kafka/HostPort.java index 7de26e6555..e9deb35a81 100644 --- a/src/jvm/storm/kafka/HostPort.java +++ b/src/jvm/storm/kafka/HostPort.java @@ -14,4 +14,15 @@ public HostPort(String host, int port) { public HostPort(String host) { this(host, 9092); } + + @Override + public boolean equals(Object o) { + HostPort other = (HostPort) o; + return host.equals(other.host) && port == other.port; + } + + @Override + public int hashCode() { + return host.hashCode(); + } } diff --git a/src/jvm/storm/kafka/KafkaConfig.java b/src/jvm/storm/kafka/KafkaConfig.java index b2a74d5ef4..03f3b779e6 100644 --- a/src/jvm/storm/kafka/KafkaConfig.java +++ b/src/jvm/storm/kafka/KafkaConfig.java @@ -7,8 +7,45 @@ import java.util.List; public class KafkaConfig implements Serializable { - public List hosts; - public int partitionsPerHost; + public static interface BrokerHosts { + + } + + public static class StaticHosts implements BrokerHosts { + public static int getNumHosts(BrokerHosts hosts) { + if(!(hosts instanceof StaticHosts)) { + throw new RuntimeException("Must use static hosts"); + } + return ((StaticHosts) hosts).hosts.size(); + } + + public List hosts; + public int partitionsPerHost; + + public static StaticHosts fromHostString(List hostStrings, int partitionsPerHost) { + return new StaticHosts(convertHosts(hostStrings), partitionsPerHost); + } + + public StaticHosts(List hosts, int partitionsPerHost) { + this.hosts = hosts; + this.partitionsPerHost = partitionsPerHost; + } + + } + + public static class ZkHosts implements BrokerHosts { + public String brokerZkStr = null; + public String brokerZkPath = null; // e.g., /kafka/brokers + public int refreshFreqSecs = 60; + + public ZkHosts(String brokerZkStr, String brokerZkPath) { + this.brokerZkStr = brokerZkStr; + this.brokerZkPath = brokerZkPath; + } + } + + + BrokerHosts hosts; public int fetchSizeBytes = 1024*1024; public int socketTimeoutMs = 10000; public int bufferSizeBytes = 1024*1024; @@ -16,18 +53,12 @@ public class KafkaConfig implements Serializable { public String topic; public long startOffsetTime = -2; public boolean forceFromStart = false; - public String brokerZkHost = null; - public String brokerZkPath = null; - public KafkaConfig(List hosts, int partitionsPerHost, String topic) { + public KafkaConfig(BrokerHosts hosts, String topic) { this.hosts = hosts; - this.partitionsPerHost = partitionsPerHost; this.topic = topic; } - public static KafkaConfig fromHostStrings(List hosts, int partitionsPerHost, String topic) { - return new KafkaConfig(convertHosts(hosts), partitionsPerHost, topic); - } public void forceStartOffsetTime(long millis) { startOffsetTime = millis; diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index c84f9d9d6c..d4912e7fc5 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -1,29 +1,19 @@ package storm.kafka; import backtype.storm.Config; -import backtype.storm.LocalCluster; import backtype.storm.spout.SpoutOutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.TopologyBuilder; import backtype.storm.topology.base.BaseRichSpout; import backtype.storm.transactional.state.TransactionalState; -import backtype.storm.utils.Utils; import java.io.Serializable; -import java.util.ArrayList; -import kafka.javaapi.consumer.SimpleConsumer; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.UUID; import org.apache.log4j.Logger; -import kafka.javaapi.message.ByteBufferMessageSet; -import kafka.message.MessageAndOffset; -import kafka.api.FetchRequest; import kafka.message.Message; +import storm.kafka.PartitionManager.KafkaMessageId; // TODO: need to add blacklisting // TODO: need to make a best effort to not re-emit messages if don't have to @@ -60,115 +50,18 @@ static enum EmitState { public static final Logger LOG = Logger.getLogger(KafkaSpout.class); - static class KafkaMessageId { - public int partition; - public long offset; - - public KafkaMessageId(int partition, long offset) { - this.partition = partition; - this.offset = offset; - } - } - - protected class PartitionManager { - Long _emittedToOffset; - SortedSet _pending = new TreeSet(); - Long _committedTo; - int _partition; - LinkedList _waitingToEmit = new LinkedList(); - - public PartitionManager(int partition) { - _partition = partition; - ZooMeta zooMeta = (ZooMeta) _state.getData(committedPath()); - SimpleConsumer consumer = _partitions.getConsumer(_partition); - int hostPartition = _partitions.getHostPartition(_partition); - - //the id stuff makes sure the spout doesn't reset the offset if it restarts - if(zooMeta==null || (!_uuid.equals(zooMeta.id) && _spoutConfig.forceFromStart)) { - _committedTo = consumer.getOffsetsBefore(_spoutConfig.topic, hostPartition, _spoutConfig.startOffsetTime, 1)[0]; - } else { - _committedTo = zooMeta.offset; - } - LOG.info("Starting Kafka " + consumer.host() + ":" + hostPartition + " from offset " + _committedTo); - _emittedToOffset = _committedTo; - } - - //returns false if it's reached the end of current batch - public EmitState next() { - if(_waitingToEmit.isEmpty()) fill(); - MessageAndRealOffset toEmit = _waitingToEmit.pollFirst(); - if(toEmit==null) { - return EmitState.NO_EMITTED; - } - List tup = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.msg.payload())); - _collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset)); - if(_waitingToEmit.size()>0) { - return EmitState.EMITTED_MORE_LEFT; - } else { - return EmitState.EMITTED_END; - } - } - - private void fill() { - SimpleConsumer consumer = _partitions.getConsumer(_partition); - int hostPartition = _partitions.getHostPartition(_partition); - LOG.info("Fetching from Kafka: " + consumer.host() + ":" + hostPartition + " from offset " + _emittedToOffset); - ByteBufferMessageSet msgs = consumer.fetch( - new FetchRequest( - _spoutConfig.topic, - hostPartition, - _emittedToOffset, - _spoutConfig.fetchSizeBytes)); - LOG.info("Fetched " + msgs.underlying().size() + " messages from Kafka: " + consumer.host() + ":" + hostPartition); - for(MessageAndOffset msg: msgs) { - _pending.add(_emittedToOffset); - _waitingToEmit.add(new MessageAndRealOffset(msg.message(), _emittedToOffset)); - _emittedToOffset = msg.offset(); - } - } - - public void ack(Long offset) { - _pending.remove(offset); - } - - public void fail(Long offset) { - //TODO: should it use in-memory ack set to skip anything that's been acked but not committed??? - // things might get crazy with lots of timeouts - if(_emittedToOffset > offset) { - _emittedToOffset = offset; - _pending.tailSet(offset).clear(); - } - } - - public void commit() { - long committedTo; - if(_pending.isEmpty()) { - committedTo = _emittedToOffset; - } else { - committedTo = _pending.first(); - } - if(committedTo!=_committedTo) { - _state.setData(committedPath(), new ZooMeta(_uuid, committedTo)); - _committedTo = committedTo; - } - } - - private String committedPath() { - return _spoutConfig.id + "/" + _partition; - } - } + String _uuid = UUID.randomUUID().toString(); SpoutConfig _spoutConfig; SpoutOutputCollector _collector; TransactionalState _state; - KafkaPartitionConnections _partitions; - Map _managers = new HashMap(); - + PartitionCoordinator _coordinator; + DynamicPartitionConnections _connections; + long _lastUpdateMs = 0; int _currPartitionIndex = 0; - List _managedPartitions = new ArrayList(); public KafkaSpout(SpoutConfig spoutConf) { _spoutConfig = spoutConf; @@ -179,6 +72,7 @@ public void open(Map conf, TopologyContext context, SpoutOutputCollector collect _collector = collector; Map stateConf = new HashMap(conf); + _connections = new DynamicPartitionConnections(_spoutConfig); List zkServers = _spoutConfig.zkServers; if(zkServers==null) zkServers = (List) conf.get(Config.STORM_ZOOKEEPER_SERVERS); @@ -196,30 +90,25 @@ public void open(Map conf, TopologyContext context, SpoutOutputCollector collect // using TransactionalState like this is a hack _state = TransactionalState.newUserState(stateConf, _spoutConfig.id, componentConf); - if(_spoutConfig.hosts == null) { - // In this case get hosts fromZK - _partitions = new ZkKafkaPartitionConnections(_spoutConfig); + int totalTasks = context.getComponentTasks(context.getThisComponentId()).size(); + if(_spoutConfig.hosts instanceof KafkaConfig.StaticHosts) { + _coordinator = new StaticCoordinator(_connections, _spoutConfig, context.getThisTaskIndex(), totalTasks, _state, _uuid); } else { - // Else just use regular partitions from host list - _partitions = new KafkaPartitionConnections(_spoutConfig); - } - - int totalPartitions = _spoutConfig.partitionsPerHost * _partitions.getNumberOfHosts(); - int numTasks = context.getComponentTasks(context.getThisComponentId()).size(); - for(int p = context.getThisTaskIndex(); p < totalPartitions; p+=numTasks) { - _managedPartitions.add(p); - _managers.put(p, new PartitionManager(p)); + _coordinator = new ZkCoordinator(_connections, conf, _spoutConfig, context.getThisTaskIndex(), totalTasks, _state, _uuid); } } @Override public void nextTuple() { - for(int i=0; i<_managedPartitions.size(); i++) { - int partition = _managedPartitions.get(_currPartitionIndex); - EmitState state = _managers.get(partition).next(); + List managers = _coordinator.getMyManagedPartitions(); + for(int i=0; i hosts = new ArrayList(); - hosts.add("localhost"); - SpoutConfig spoutConf = SpoutConfig.fromHostStrings(hosts, 8, "clicks", "/kafkastorm", "id"); - spoutConf.scheme = new StringScheme(); - spoutConf.forceStartOffsetTime(-2); - - // spoutConf.zkServers = new ArrayList() {{ - // add("localhost"); - // }}; - // spoutConf.zkPort = 2181; - - builder.setSpout("spout", new KafkaSpout(spoutConf), 3); - - Config conf = new Config(); - //conf.setDebug(true); - - LocalCluster cluster = new LocalCluster(); - cluster.submitTopology("kafka-test", conf, builder.createTopology()); - - Utils.sleep(600000); +// TopologyBuilder builder = new TopologyBuilder(); +// List hosts = new ArrayList(); +// hosts.add("localhost"); +// SpoutConfig spoutConf = SpoutConfig.fromHostStrings(hosts, 8, "clicks", "/kafkastorm", "id"); +// spoutConf.scheme = new StringScheme(); +// spoutConf.forceStartOffsetTime(-2); +// +// // spoutConf.zkServers = new ArrayList() {{ +// // add("localhost"); +// // }}; +// // spoutConf.zkPort = 2181; +// +// builder.setSpout("spout", new KafkaSpout(spoutConf), 3); +// +// Config conf = new Config(); +// //conf.setDebug(true); +// +// LocalCluster cluster = new LocalCluster(); +// cluster.submitTopology("kafka-test", conf, builder.createTopology()); +// +// Utils.sleep(600000); } } diff --git a/src/jvm/storm/kafka/KafkaUtils.java b/src/jvm/storm/kafka/KafkaUtils.java index 193d9f3c17..e0f4151f61 100644 --- a/src/jvm/storm/kafka/KafkaUtils.java +++ b/src/jvm/storm/kafka/KafkaUtils.java @@ -11,18 +11,20 @@ import kafka.javaapi.message.ByteBufferMessageSet; import kafka.message.Message; import kafka.message.MessageAndOffset; +import storm.kafka.KafkaConfig.StaticHosts; public class KafkaUtils { public static BatchMeta emitPartitionBatchNew(KafkaConfig config, int partition, SimpleConsumer consumer, TransactionAttempt attempt, BatchOutputCollector collector, BatchMeta lastMeta) { + StaticHosts hosts = (StaticHosts) config.hosts; long offset = 0; if(lastMeta!=null) { offset = lastMeta.nextOffset; } ByteBufferMessageSet msgs; try { - msgs = consumer.fetch(new FetchRequest(config.topic, partition % config.partitionsPerHost, offset, config.fetchSizeBytes)); + msgs = consumer.fetch(new FetchRequest(config.topic, partition % hosts.partitionsPerHost, offset, config.fetchSizeBytes)); } catch(Exception e) { if(e instanceof ConnectException) { throw new FailedFetchException(e); diff --git a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java index b7a853415a..cc78cf5257 100644 --- a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java +++ b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java @@ -11,6 +11,7 @@ import java.util.Map; import kafka.javaapi.consumer.SimpleConsumer; import org.apache.log4j.Logger; +import storm.kafka.KafkaConfig.StaticHosts; public class OpaqueTransactionalKafkaSpout implements IOpaquePartitionedTransactionalSpout { @@ -61,10 +62,10 @@ public void close() { } class Emitter implements IOpaquePartitionedTransactionalSpout.Emitter { - KafkaPartitionConnections _connections; + StaticPartitionConnections _connections; public Emitter() { - _connections = new KafkaPartitionConnections(_config); + _connections = new StaticPartitionConnections(_config); } @Override @@ -87,7 +88,8 @@ public BatchMeta emitPartitionBatch(TransactionAttempt attempt, BatchOutputColle @Override public int numPartitions() { - return _config.hosts.size() * _config.partitionsPerHost; + StaticHosts hosts = (StaticHosts) _config.hosts; + return hosts.hosts.size() * hosts.partitionsPerHost; } @Override diff --git a/src/jvm/storm/kafka/PartitionCoordinator.java b/src/jvm/storm/kafka/PartitionCoordinator.java new file mode 100644 index 0000000000..21f71cd8d7 --- /dev/null +++ b/src/jvm/storm/kafka/PartitionCoordinator.java @@ -0,0 +1,8 @@ +package storm.kafka; + +import java.util.List; + +public interface PartitionCoordinator { + List getMyManagedPartitions(); + PartitionManager getManager(GlobalPartitionId id); +} diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java new file mode 100644 index 0000000000..6288e8660d --- /dev/null +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -0,0 +1,126 @@ +package storm.kafka; + +import backtype.storm.spout.SpoutOutputCollector; +import backtype.storm.transactional.state.TransactionalState; +import backtype.storm.utils.Utils; +import java.util.LinkedList; +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; +import kafka.api.FetchRequest; +import kafka.javaapi.consumer.SimpleConsumer; +import kafka.javaapi.message.ByteBufferMessageSet; +import kafka.message.MessageAndOffset; +import org.apache.log4j.Logger; +import storm.kafka.KafkaSpout.EmitState; +import storm.kafka.KafkaSpout.MessageAndRealOffset; +import storm.kafka.KafkaSpout.ZooMeta; + +public class PartitionManager { + public static final Logger LOG = Logger.getLogger(PartitionManager.class); + + static class KafkaMessageId { + public GlobalPartitionId partition; + public long offset; + + public KafkaMessageId(GlobalPartitionId partition, long offset) { + this.partition = partition; + this.offset = offset; + } + } + + Long _emittedToOffset; + SortedSet _pending = new TreeSet(); + Long _committedTo; + LinkedList _waitingToEmit = new LinkedList(); + TransactionalState _state; + GlobalPartitionId _partition; + SpoutConfig _spoutConfig; + String _topologyInstanceId; + SimpleConsumer _consumer; + DynamicPartitionConnections _connections; + + public PartitionManager(DynamicPartitionConnections connections, String topologyInstanceId, SpoutConfig spoutConfig, TransactionalState state, GlobalPartitionId id) { + _state = state; + _connections = connections; + _spoutConfig = spoutConfig; + _topologyInstanceId = topologyInstanceId; + ZooMeta zooMeta = (ZooMeta) _state.getData(committedPath()); + _consumer = connections.register(id.host, id.partition); + + //the id stuff makes sure the spout doesn't reset the offset if it restarts + if(zooMeta==null || (!topologyInstanceId.equals(zooMeta.id) && spoutConfig.forceFromStart)) { + _committedTo = _consumer.getOffsetsBefore(spoutConfig.topic, id.partition, spoutConfig.startOffsetTime, 1)[0]; + } else { + _committedTo = zooMeta.offset; + } + LOG.info("Starting Kafka " + _consumer.host() + ":" + id.partition + " from offset " + _committedTo); + _emittedToOffset = _committedTo; + } + + //returns false if it's reached the end of current batch + public EmitState next(SpoutOutputCollector collector) { + if(_waitingToEmit.isEmpty()) fill(); + MessageAndRealOffset toEmit = _waitingToEmit.pollFirst(); + if(toEmit==null) { + return EmitState.NO_EMITTED; + } + List tup = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.msg.payload())); + collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset)); + if(_waitingToEmit.size()>0) { + return EmitState.EMITTED_MORE_LEFT; + } else { + return EmitState.EMITTED_END; + } + } + + private void fill() { + LOG.info("Fetching from Kafka: " + _consumer.host() + ":" + _partition.partition + " from offset " + _emittedToOffset); + ByteBufferMessageSet msgs = _consumer.fetch( + new FetchRequest( + _spoutConfig.topic, + _partition.partition, + _emittedToOffset, + _spoutConfig.fetchSizeBytes)); + LOG.info("Fetched " + msgs.underlying().size() + " messages from Kafka: " + _consumer.host() + ":" + _partition.partition); + for(MessageAndOffset msg: msgs) { + _pending.add(_emittedToOffset); + _waitingToEmit.add(new MessageAndRealOffset(msg.message(), _emittedToOffset)); + _emittedToOffset = msg.offset(); + } + } + + public void ack(Long offset) { + _pending.remove(offset); + } + + public void fail(Long offset) { + //TODO: should it use in-memory ack set to skip anything that's been acked but not committed??? + // things might get crazy with lots of timeouts + if(_emittedToOffset > offset) { + _emittedToOffset = offset; + _pending.tailSet(offset).clear(); + } + } + + public void commit() { + long committedTo; + if(_pending.isEmpty()) { + committedTo = _emittedToOffset; + } else { + committedTo = _pending.first(); + } + if(committedTo!=_committedTo) { + _state.setData(committedPath(), new ZooMeta(_topologyInstanceId, committedTo)); + _committedTo = committedTo; + } + } + + private String committedPath() { + return _spoutConfig.id + "/" + _partition; + } + + public void close() { + _connections.unregister(_partition.host, _partition.partition); + } +} \ No newline at end of file diff --git a/src/jvm/storm/kafka/SpoutConfig.java b/src/jvm/storm/kafka/SpoutConfig.java index fbce47fbab..05551ece2b 100644 --- a/src/jvm/storm/kafka/SpoutConfig.java +++ b/src/jvm/storm/kafka/SpoutConfig.java @@ -11,20 +11,9 @@ public class SpoutConfig extends KafkaConfig implements Serializable { public String id = null; public long stateUpdateIntervalMs = 2000; - public SpoutConfig(List hosts, int partitionsPerHost, String topic, String zkRoot, String id) { - super(hosts, partitionsPerHost, topic); + public SpoutConfig(BrokerHosts hosts, String topic, String zkRoot, String id) { + super(hosts, topic); this.zkRoot = zkRoot; this.id = id; } - - public static SpoutConfig fromHostStrings(List hosts, int partitionsPerHost, String topic, String zkRoot, String id) { - return new SpoutConfig(KafkaConfig.convertHosts(hosts), partitionsPerHost, topic, zkRoot, id); - } - - public static SpoutConfig fromZkServer(String brokerZkHost, String brokerZkPath, int partitionsPerHost, String topic, String zkRoot, String id) { - SpoutConfig conf = new SpoutConfig(null, partitionsPerHost, topic, zkRoot, id); - conf.brokerZkHost = brokerZkHost; - conf.brokerZkPath = brokerZkPath; - return conf; - } } diff --git a/src/jvm/storm/kafka/StaticCoordinator.java b/src/jvm/storm/kafka/StaticCoordinator.java new file mode 100644 index 0000000000..c8dd9d5945 --- /dev/null +++ b/src/jvm/storm/kafka/StaticCoordinator.java @@ -0,0 +1,41 @@ +package storm.kafka; + +import backtype.storm.transactional.state.TransactionalState; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import storm.kafka.KafkaConfig.StaticHosts; + + +public class StaticCoordinator implements PartitionCoordinator { + Map _managers = new HashMap(); + List _allManagers = new ArrayList(); + + public StaticCoordinator(DynamicPartitionConnections connections, SpoutConfig config, int taskIndex, int totalTasks, TransactionalState state, String topologyInstanceId) { + StaticHosts hosts = (StaticHosts) config.hosts; + List allPartitionIds = new ArrayList(); + for(HostPort h: hosts.hosts) { + for(int i=0; i getMyManagedPartitions() { + return _allManagers; + } + + public PartitionManager getManager(GlobalPartitionId id) { + return _managers.get(id); + } + +} diff --git a/src/jvm/storm/kafka/KafkaPartitionConnections.java b/src/jvm/storm/kafka/StaticPartitionConnections.java similarity index 56% rename from src/jvm/storm/kafka/KafkaPartitionConnections.java rename to src/jvm/storm/kafka/StaticPartitionConnections.java index ce9835102e..624b2de50e 100644 --- a/src/jvm/storm/kafka/KafkaPartitionConnections.java +++ b/src/jvm/storm/kafka/StaticPartitionConnections.java @@ -1,21 +1,28 @@ package storm.kafka; import java.util.HashMap; +import java.util.List; import java.util.Map; import kafka.javaapi.consumer.SimpleConsumer; +import storm.kafka.KafkaConfig.StaticHosts; -public class KafkaPartitionConnections { +public class StaticPartitionConnections { Map _kafka = new HashMap(); KafkaConfig _config; - - public KafkaPartitionConnections(KafkaConfig conf) { + StaticHosts hosts; + + public StaticPartitionConnections(KafkaConfig conf) { _config = conf; + if(!(conf.hosts instanceof KafkaConfig.StaticHosts)) { + throw new RuntimeException("Must configure with static hosts"); + } + this.hosts = (StaticHosts) conf.hosts; } public SimpleConsumer getConsumer(int partition) { - int hostIndex = partition / _config.partitionsPerHost; + int hostIndex = partition / hosts.partitionsPerHost; if(!_kafka.containsKey(hostIndex)) { - HostPort hp = _config.hosts.get(hostIndex); + HostPort hp = hosts.hosts.get(hostIndex); _kafka.put(hostIndex, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); } @@ -23,11 +30,11 @@ public SimpleConsumer getConsumer(int partition) { } public int getHostPartition(int globalPartition) { - return globalPartition % _config.partitionsPerHost; + return globalPartition % hosts.partitionsPerHost; } public int getNumberOfHosts() { - return _config.hosts.size(); + return hosts.hosts.size(); } public void close() { diff --git a/src/jvm/storm/kafka/TestTopology.java b/src/jvm/storm/kafka/TestTopology.java index 0c0e1644cf..74bbc8fd3e 100644 --- a/src/jvm/storm/kafka/TestTopology.java +++ b/src/jvm/storm/kafka/TestTopology.java @@ -9,6 +9,7 @@ import backtype.storm.tuple.Tuple; import java.util.ArrayList; import java.util.List; +import storm.kafka.KafkaConfig.StaticHosts; public class TestTopology { public static class PrinterBolt extends BaseBasicBolt { @@ -26,7 +27,7 @@ public void execute(Tuple tuple, BasicOutputCollector collector) { public static void main(String [] args) throws Exception { List hosts = new ArrayList(); hosts.add("localhost"); - KafkaConfig kafkaConf = KafkaConfig.fromHostStrings(hosts, 3, "test"); + KafkaConfig kafkaConf = new KafkaConfig(StaticHosts.fromHostString(hosts, 3), "test"); kafkaConf.scheme = new StringScheme(); LocalCluster cluster = new LocalCluster(); TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("id", "spout", diff --git a/src/jvm/storm/kafka/TransactionalKafkaSpout.java b/src/jvm/storm/kafka/TransactionalKafkaSpout.java index e28e898f53..d5b6e10412 100644 --- a/src/jvm/storm/kafka/TransactionalKafkaSpout.java +++ b/src/jvm/storm/kafka/TransactionalKafkaSpout.java @@ -14,6 +14,7 @@ import kafka.javaapi.consumer.SimpleConsumer; import kafka.javaapi.message.ByteBufferMessageSet; import kafka.message.MessageAndOffset; +import storm.kafka.KafkaConfig.StaticHosts; public class TransactionalKafkaSpout extends BasePartitionedTransactionalSpout { @@ -43,10 +44,13 @@ public boolean isReady() { } class Emitter implements IPartitionedTransactionalSpout.Emitter { - KafkaPartitionConnections _connections; + StaticPartitionConnections _connections; + int partitionsPerHost; public Emitter() { - _connections = new KafkaPartitionConnections(_config); + _connections = new StaticPartitionConnections(_config); + StaticHosts hosts = (StaticHosts) _config.hosts; + partitionsPerHost = hosts.partitionsPerHost; } @Override @@ -60,7 +64,7 @@ public BatchMeta emitPartitionBatchNew(TransactionAttempt attempt, BatchOutputCo public void emitPartitionBatch(TransactionAttempt attempt, BatchOutputCollector collector, int partition, BatchMeta meta) { SimpleConsumer consumer = _connections.getConsumer(partition); - ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition % _config.partitionsPerHost, meta.offset, _config.fetchSizeBytes)); + ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition % partitionsPerHost, meta.offset, _config.fetchSizeBytes)); long offset = meta.offset; for(MessageAndOffset msg: msgs) { if(offset == meta.nextOffset) break; @@ -97,7 +101,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } private int computeNumPartitions() { - return _config.hosts.size() * _config.partitionsPerHost; + StaticHosts hosts = (StaticHosts) _config.hosts; + return hosts.hosts.size() * hosts.partitionsPerHost; } @Override diff --git a/src/jvm/storm/kafka/ZkCoordinator.java b/src/jvm/storm/kafka/ZkCoordinator.java new file mode 100644 index 0000000000..9791037f45 --- /dev/null +++ b/src/jvm/storm/kafka/ZkCoordinator.java @@ -0,0 +1,147 @@ +package storm.kafka; + +import backtype.storm.Config; +import backtype.storm.transactional.state.TransactionalState; +import backtype.storm.utils.Utils; +import com.netflix.curator.framework.CuratorFramework; +import com.netflix.curator.framework.CuratorFrameworkFactory; +import com.netflix.curator.retry.RetryNTimes; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import kafka.javaapi.consumer.SimpleConsumer; +import storm.kafka.KafkaConfig.ZkHosts; + +public class ZkCoordinator implements PartitionCoordinator { + SpoutConfig _config; + int _taskIndex; + int _totalTasks; + TransactionalState _state; + String _topologyInstanceId; + CuratorFramework _curator; + Map _managers = new HashMap(); + List _cachedList; + Long _lastRefreshTime = null; + int _refreshFreqMs; + ZkHosts _brokerConf; + DynamicPartitionConnections _connections; + + public ZkCoordinator(DynamicPartitionConnections connections, Map conf, SpoutConfig config, int taskIndex, int totalTasks, TransactionalState state, String topologyInstanceId) { + _config = config; + _connections = connections; + _taskIndex = taskIndex; + _totalTasks = totalTasks; + _state = state; + _topologyInstanceId = topologyInstanceId; + + + _brokerConf = (ZkHosts) config.hosts; + _refreshFreqMs = _brokerConf.refreshFreqSecs * 1000; + try { + _curator = CuratorFrameworkFactory.newClient( + _brokerConf.brokerZkStr, + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), + 15000, + new RetryNTimes(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + @Override + public List getMyManagedPartitions() { + if(_lastRefreshTime==null || (System.currentTimeMillis() - _lastRefreshTime) > _refreshFreqMs) { + refresh(); + } + return _cachedList; + } + + void refresh() { + try { + String topicBrokersPath = _brokerConf.brokerZkPath + "/topics/" + _config.topic; + String brokerInfoPath = _brokerConf.brokerZkPath + "/ids"; + List children = _curator.getChildren().forPath(topicBrokersPath); + + Set mine = new HashSet(); + for(String c: children) { + try { + byte[] numPartitionsData = _curator.getData().forPath(topicBrokersPath + "/" + c); + byte[] hostPortData = _curator.getData().forPath(brokerInfoPath + "/" + c); + + + + HostPort hp = getBrokerHost(hostPortData); + int numPartitions = getNumPartitions(numPartitionsData); + + for(int i=0; i curr = _managers.keySet(); + Set newPartitions = new HashSet(mine); + newPartitions.removeAll(curr); + + Set deletedPartitions = new HashSet(curr); + deletedPartitions.removeAll(mine); + + for(GlobalPartitionId id: deletedPartitions) { + PartitionManager man = _managers.remove(id); + man.close(); + } + + for(GlobalPartitionId id: newPartitions) { + SimpleConsumer consumer = _connections.register(id.host, id.partition); + PartitionManager man = new PartitionManager(_connections, _topologyInstanceId, _config, _state, id); + _managers.put(id, man); + } + + } catch(Exception e) { + throw new RuntimeException(e); + } + _cachedList = new ArrayList(_managers.values()); + } + + @Override + public PartitionManager getManager(GlobalPartitionId id) { + return _managers.get(id); + } + + private boolean myOwnership(GlobalPartitionId id) { + int val = id.host.hashCode() + 23 * id.partition; + return val % _totalTasks == _taskIndex; + } + + + + private static HostPort getBrokerHost(byte[] contents) { + try { + String[] hostString = new String(contents, "UTF-8").split(":"); + String host = hostString[hostString.length - 2]; + int port = Integer.parseInt(hostString[hostString.length - 1]); + return new HostPort(host, port); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + private static int getNumPartitions(byte[] contents) { + try { + return Integer.parseInt(new String(contents, "UTF-8")); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java b/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java deleted file mode 100644 index c5d3491d26..0000000000 --- a/src/jvm/storm/kafka/ZkKafkaPartitionConnections.java +++ /dev/null @@ -1,104 +0,0 @@ -package storm.kafka; - -import java.io.IOException; -import java.lang.InterruptedException; -import java.lang.RuntimeException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import kafka.javaapi.consumer.SimpleConsumer; - -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.Watcher.Event.EventType; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.Stat; - -public class ZkKafkaPartitionConnections extends KafkaPartitionConnections { - private static int ZK_TIMEOUT = 10000; - - Map _pathToHostIndex = new HashMap(); - ZooKeeper _zkClient; - String _brokerZkPath; - - // TODO: Only handles node data changes. - // Should we handle the case where new producers enter or - // old ones leave? - Watcher eventWatcher = new Watcher() { - @Override - public void process(WatchedEvent event) { - if(event.getType() == EventType.NodeDataChanged) { - String path = event.getPath(); - HostPort hp = getBrokerHost(path); - int hostIndex = _pathToHostIndex.get(path); - _kafka.put(hostIndex, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); - } else { - throw new RuntimeException("Kafka producer node changed but don't know how to deal with event: " + event.getType()); - } - } - }; - - public ZkKafkaPartitionConnections(KafkaConfig conf) { - super(conf); - _brokerZkPath = conf.brokerZkPath; - try { - _zkClient = new ZooKeeper(conf.brokerZkHost, 2000, eventWatcher); - } catch(IOException e) { - // Screw you checked exceptions! - throw new RuntimeException("Error while creating zkClient: " + e); - } - setupHosts(); - } - - @Override - public int getNumberOfHosts() { - return _kafka.size(); - } - - @Override - public SimpleConsumer getConsumer(int partition) { - int hostIndex = partition / _config.partitionsPerHost; - return _kafka.get(hostIndex); - } - - @Override - public void close() { - super.close(); - try { - _zkClient.close(); - } catch(InterruptedException e) { - throw new RuntimeException("Error while closing client"); - } - } - - private HostPort getBrokerHost(String zkPath) { - try { - String zkData = new String(_zkClient.getData(_brokerZkPath + "/" + zkPath, true, new Stat())); - String[] hostString = zkData.split(":"); - String host = hostString[hostString.length - 2]; - int port = Integer.parseInt(hostString[hostString.length - 1]); - return new HostPort(host, port); - } catch(KeeperException e) { - throw new RuntimeException("Could not get broker data: " + e); - } catch(InterruptedException e) { - throw new RuntimeException("Error while creating zkClient: " + e); - } - } - - private void setupHosts() { - try { - List brokers = _zkClient.getChildren(_brokerZkPath, false); - for(int i = 0; i < brokers.size(); i++) { - String path = brokers.get(i); - HostPort hp = getBrokerHost(path); - _kafka.put(i, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes)); - } - } catch(KeeperException e) { - throw new RuntimeException("Could not get brokers: " + e); - } catch(InterruptedException e) { - throw new RuntimeException("Error while creating zkClient: " + e); - } - } -} From b7ba9d9455582b8ef180b96d7f543907b74b674f Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 2 Jul 2012 18:42:24 -0700 Subject: [PATCH 103/349] make brokerhosts serializable --- src/jvm/storm/kafka/KafkaConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jvm/storm/kafka/KafkaConfig.java b/src/jvm/storm/kafka/KafkaConfig.java index 03f3b779e6..81ccc1df28 100644 --- a/src/jvm/storm/kafka/KafkaConfig.java +++ b/src/jvm/storm/kafka/KafkaConfig.java @@ -7,7 +7,7 @@ import java.util.List; public class KafkaConfig implements Serializable { - public static interface BrokerHosts { + public static interface BrokerHosts extends Serializable { } From e4305171091ee31c906f4cac7e476fe7f7681e78 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 2 Jul 2012 18:46:06 -0700 Subject: [PATCH 104/349] start curator client --- src/jvm/storm/kafka/ZkCoordinator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/jvm/storm/kafka/ZkCoordinator.java b/src/jvm/storm/kafka/ZkCoordinator.java index 9791037f45..7c5068bf78 100644 --- a/src/jvm/storm/kafka/ZkCoordinator.java +++ b/src/jvm/storm/kafka/ZkCoordinator.java @@ -49,6 +49,7 @@ public ZkCoordinator(DynamicPartitionConnections connections, Map conf, SpoutCon 15000, new RetryNTimes(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); + _curator.start(); } catch (IOException e) { throw new RuntimeException(e); } From c49b687e37c323e559621b7d058de1a1e5d5eaf5 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 2 Jul 2012 18:55:07 -0700 Subject: [PATCH 105/349] got it working --- src/jvm/storm/kafka/PartitionManager.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 6288e8660d..e03db3c90a 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -42,6 +42,7 @@ public KafkaMessageId(GlobalPartitionId partition, long offset) { public PartitionManager(DynamicPartitionConnections connections, String topologyInstanceId, SpoutConfig spoutConfig, TransactionalState state, GlobalPartitionId id) { _state = state; + _partition = id; _connections = connections; _spoutConfig = spoutConfig; _topologyInstanceId = topologyInstanceId; From 31b9e928604be16d8cc648aa8ba4afa614e66256 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 11 Jul 2012 02:28:46 -0700 Subject: [PATCH 106/349] change version of dynamic kafka spout --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 27a8e9ae8c..6ac93e1151 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.7.3-SNAPSHOT" +(defproject storm/storm-kafka "0.7.3-dynamic-SNAPSHOT" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} From 887cad498014e584b7a6eca79357962bdc9a040e Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 12 Jul 2012 13:54:37 -0700 Subject: [PATCH 107/349] added more logging into spout, fixed bug where it would refresh zk on every nextTuple --- project.clj | 2 +- src/jvm/storm/kafka/GlobalPartitionId.java | 7 +++++-- src/jvm/storm/kafka/HostPort.java | 9 ++++++++- src/jvm/storm/kafka/PartitionManager.java | 7 ++++++- src/jvm/storm/kafka/ZkCoordinator.java | 10 +++++++++- 5 files changed, 29 insertions(+), 6 deletions(-) diff --git a/project.clj b/project.clj index 6ac93e1151..f84546921a 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.7.3-dynamic-SNAPSHOT" +(defproject storm/storm-kafka "0.7.3-dynamic2-SNAPSHOT" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} diff --git a/src/jvm/storm/kafka/GlobalPartitionId.java b/src/jvm/storm/kafka/GlobalPartitionId.java index bbe04cf631..460fc9a8b5 100644 --- a/src/jvm/storm/kafka/GlobalPartitionId.java +++ b/src/jvm/storm/kafka/GlobalPartitionId.java @@ -20,6 +20,9 @@ public boolean equals(Object o) { public int hashCode() { return 13 * host.hashCode() + partition; } - - + + @Override + public String toString() { + return host.toString() + ":" + partition; + } } diff --git a/src/jvm/storm/kafka/HostPort.java b/src/jvm/storm/kafka/HostPort.java index e9deb35a81..ccc1183ea6 100644 --- a/src/jvm/storm/kafka/HostPort.java +++ b/src/jvm/storm/kafka/HostPort.java @@ -24,5 +24,12 @@ public boolean equals(Object o) { @Override public int hashCode() { return host.hashCode(); - } + } + + @Override + public String toString() { + return host + ":" + port; + } + + } diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index e03db3c90a..3221a146e5 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -68,7 +68,7 @@ public EmitState next(SpoutOutputCollector collector) { } List tup = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.msg.payload())); collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset)); - if(_waitingToEmit.size()>0) { + if(!_waitingToEmit.isEmpty()) { return EmitState.EMITTED_MORE_LEFT; } else { return EmitState.EMITTED_END; @@ -89,6 +89,7 @@ private void fill() { _waitingToEmit.add(new MessageAndRealOffset(msg.message(), _emittedToOffset)); _emittedToOffset = msg.offset(); } + LOG.info("Added " + msgs.underlying().size() + " messages from Kafka: " + _consumer.host() + ":" + _partition.partition + " to internal buffers"); } public void ack(Long offset) { @@ -105,6 +106,7 @@ public void fail(Long offset) { } public void commit() { + LOG.info("Committing offset for " + _partition); long committedTo; if(_pending.isEmpty()) { committedTo = _emittedToOffset; @@ -112,9 +114,12 @@ public void commit() { committedTo = _pending.first(); } if(committedTo!=_committedTo) { + LOG.info("Writing committed offset to ZK: " + committedTo); _state.setData(committedPath(), new ZooMeta(_topologyInstanceId, committedTo)); + LOG.info("Wrote committed offset to ZK: " + committedTo); _committedTo = committedTo; } + LOG.info("Comitted offset for " + _partition); } private String committedPath() { diff --git a/src/jvm/storm/kafka/ZkCoordinator.java b/src/jvm/storm/kafka/ZkCoordinator.java index 7c5068bf78..b06975da37 100644 --- a/src/jvm/storm/kafka/ZkCoordinator.java +++ b/src/jvm/storm/kafka/ZkCoordinator.java @@ -15,9 +15,12 @@ import java.util.Map; import java.util.Set; import kafka.javaapi.consumer.SimpleConsumer; +import org.apache.log4j.Logger; import storm.kafka.KafkaConfig.ZkHosts; public class ZkCoordinator implements PartitionCoordinator { + public static Logger LOG = Logger.getLogger(ZkCoordinator.class); + SpoutConfig _config; int _taskIndex; int _totalTasks; @@ -60,12 +63,14 @@ public ZkCoordinator(DynamicPartitionConnections connections, Map conf, SpoutCon public List getMyManagedPartitions() { if(_lastRefreshTime==null || (System.currentTimeMillis() - _lastRefreshTime) > _refreshFreqMs) { refresh(); + _lastRefreshTime = System.currentTimeMillis(); } return _cachedList; } void refresh() { try { + LOG.info("Refreshing partition manager connections"); String topicBrokersPath = _brokerConf.brokerZkPath + "/topics/" + _config.topic; String brokerInfoPath = _brokerConf.brokerZkPath + "/ids"; List children = _curator.getChildren().forPath(topicBrokersPath); @@ -98,13 +103,15 @@ void refresh() { Set deletedPartitions = new HashSet(curr); deletedPartitions.removeAll(mine); + LOG.info("Deleted partition managers: " + deletedPartitions.toString()); + for(GlobalPartitionId id: deletedPartitions) { PartitionManager man = _managers.remove(id); man.close(); } + LOG.info("New partition managers: " + newPartitions.toString()); for(GlobalPartitionId id: newPartitions) { - SimpleConsumer consumer = _connections.register(id.host, id.partition); PartitionManager man = new PartitionManager(_connections, _topologyInstanceId, _config, _state, id); _managers.put(id, man); } @@ -113,6 +120,7 @@ void refresh() { throw new RuntimeException(e); } _cachedList = new ArrayList(_managers.values()); + LOG.info("Finished refreshing"); } @Override From 7b5264967c855f353f4ac4f333d05a0e721f91a0 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 12 Jul 2012 13:54:50 -0700 Subject: [PATCH 108/349] fix version --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index f84546921a..6ac93e1151 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.7.3-dynamic2-SNAPSHOT" +(defproject storm/storm-kafka "0.7.3-dynamic-SNAPSHOT" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} From d2c73f85c579f65266ba87c1efd7d07ae6735eef Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 12 Jul 2012 14:10:26 -0700 Subject: [PATCH 109/349] make it possible to do filtering with the spout scheme --- src/jvm/storm/kafka/PartitionManager.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 3221a146e5..f58d497a47 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -62,12 +62,19 @@ public PartitionManager(DynamicPartitionConnections connections, String topology //returns false if it's reached the end of current batch public EmitState next(SpoutOutputCollector collector) { if(_waitingToEmit.isEmpty()) fill(); - MessageAndRealOffset toEmit = _waitingToEmit.pollFirst(); - if(toEmit==null) { - return EmitState.NO_EMITTED; + while(true) { + MessageAndRealOffset toEmit = _waitingToEmit.pollFirst(); + if(toEmit==null) { + return EmitState.NO_EMITTED; + } + List tup = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.msg.payload())); + if(tup!=null) { + collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset)); + break; + } else { + ack(toEmit.offset); + } } - List tup = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.msg.payload())); - collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset)); if(!_waitingToEmit.isEmpty()) { return EmitState.EMITTED_MORE_LEFT; } else { From f75e9ce3e7ef27aa29572d4a59ee5fd626542b82 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 20 Jul 2012 12:07:59 -0700 Subject: [PATCH 110/349] added example of manual drpc --- src/jvm/storm/starter/ManualDRPC.java | 53 +++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/jvm/storm/starter/ManualDRPC.java diff --git a/src/jvm/storm/starter/ManualDRPC.java b/src/jvm/storm/starter/ManualDRPC.java new file mode 100644 index 0000000000..fd45c9c0c6 --- /dev/null +++ b/src/jvm/storm/starter/ManualDRPC.java @@ -0,0 +1,53 @@ +package storm.starter; + +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.LocalDRPC; +import backtype.storm.drpc.DRPCSpout; +import backtype.storm.drpc.ReturnResults; +import backtype.storm.topology.BasicOutputCollector; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.TopologyBuilder; +import backtype.storm.topology.base.BaseBasicBolt; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; + + +public class ManualDRPC { + public static class ExclamationBolt extends BaseBasicBolt { + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("result", "return-info")); + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + String arg = tuple.getString(0); + Object retInfo = tuple.getValue(1); + collector.emit(new Values(arg + "!!!", retInfo)); + } + + } + + public static void main(String[] args) { + TopologyBuilder builder = new TopologyBuilder(); + LocalDRPC drpc = new LocalDRPC(); + + DRPCSpout spout = new DRPCSpout("exclamation", drpc); + builder.setSpout("drpc", spout); + builder.setBolt("exclaim", new ExclamationBolt(), 3) + .shuffleGrouping("drpc"); + builder.setBolt("return", new ReturnResults(), 3) + .shuffleGrouping("exclaim"); + + LocalCluster cluster = new LocalCluster(); + Config conf = new Config(); + cluster.submitTopology("exclaim", conf, builder.createTopology()); + + System.out.println(drpc.execute("exclamation", "aaa")); + System.out.println(drpc.execute("exclamation", "bbb")); + + } +} From 63e1953d34e96e17d39d5e5a1502669a395b61e6 Mon Sep 17 00:00:00 2001 From: Bart Olsthoorn Date: Sat, 21 Jul 2012 13:25:38 +0200 Subject: [PATCH 111/349] ruby splitsentence bolt --- multilang/resources/splitsentence.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 multilang/resources/splitsentence.rb diff --git a/multilang/resources/splitsentence.rb b/multilang/resources/splitsentence.rb new file mode 100644 index 0000000000..6b719ed099 --- /dev/null +++ b/multilang/resources/splitsentence.rb @@ -0,0 +1,11 @@ +require "./storm" + +class SplitSentenceBolt < Storm::Bolt + def process(tup) + tup.values[0].split(" ").each do |word| + emit([word]) + end + end +end + +SplitSentenceBolt.new.run \ No newline at end of file From 237d716003192d4ceea85e8526b25dcd38f9f49d Mon Sep 17 00:00:00 2001 From: tombrown52 Date: Wed, 25 Jul 2012 10:25:22 -0600 Subject: [PATCH 112/349] Update storm-kafka/src/jvm/storm/kafka/ZkCoordinator.java Fixing a bug that causes partitions to be unassigned if the hash code of the "HostPort" is negative. --- src/jvm/storm/kafka/ZkCoordinator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jvm/storm/kafka/ZkCoordinator.java b/src/jvm/storm/kafka/ZkCoordinator.java index b06975da37..f287d8070b 100644 --- a/src/jvm/storm/kafka/ZkCoordinator.java +++ b/src/jvm/storm/kafka/ZkCoordinator.java @@ -129,7 +129,7 @@ public PartitionManager getManager(GlobalPartitionId id) { } private boolean myOwnership(GlobalPartitionId id) { - int val = id.host.hashCode() + 23 * id.partition; + int val = Math.abs(id.host.hashCode() + 23 * id.partition); return val % _totalTasks == _taskIndex; } From c67ab70e4b8436094a743e1cc782892e8b8615e5 Mon Sep 17 00:00:00 2001 From: Brian O'Neill Date: Mon, 30 Jul 2012 23:25:05 -0400 Subject: [PATCH 113/349] Pom file for maven support. --- pom.xml | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 pom.xml diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000000..c20a9e9145 --- /dev/null +++ b/pom.xml @@ -0,0 +1,76 @@ + + + + org.sonatype.oss + oss-parent + 7 + + + 4.0.0 + storm + storm-kafka + 0.72.1-SNAPSHOT + Storm Kafka Support + Storm Kafka Support + + + + Eclipse Public License - v 1.0 + http://www.eclipse.org/legal/epl-v10.html + repo + + + + scm:git:git@github.com:boneill42/storm-kafka.git + scm:git:git@github.com:boneill42/storm-kafka.git + :git@github.com:boneill42/storm-cassandra.git + + + + + boneill42 + Brian O'Neill + bone@alumni.brown.edu + + + + + + clojars.org + http://clojars.org/repo + + + + + + org.apache.kafka + kafka + 0.7.1 + + + org.scala-lang + scala-library + 2.8.0 + + + storm + storm + 0.7.2 + + + + src/jvm + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.6 + 1.6 + + + + + + + From 0cf4a4ef91a7bad3256347c412a42f43fa71a744 Mon Sep 17 00:00:00 2001 From: Jason Jackson Date: Mon, 30 Jul 2012 15:30:42 -0700 Subject: [PATCH 114/349] write kafka spout zookeeper state in JSON format --- src/jvm/storm/kafka/KafkaSpout.java | 41 ++++------------------ src/jvm/storm/kafka/KafkaSpoutState.java | 29 +++++++++++++++ src/jvm/storm/kafka/PartitionManager.java | 35 +++++++++--------- src/jvm/storm/kafka/StaticCoordinator.java | 5 ++- src/jvm/storm/kafka/ZkCoordinator.java | 19 +++++----- 5 files changed, 63 insertions(+), 66 deletions(-) create mode 100644 src/jvm/storm/kafka/KafkaSpoutState.java diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index d4912e7fc5..d23b32e74d 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -5,7 +5,6 @@ import backtype.storm.task.TopologyContext; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseRichSpout; -import backtype.storm.transactional.state.TransactionalState; import java.io.Serializable; import java.util.HashMap; import java.util.List; @@ -18,20 +17,6 @@ // TODO: need to add blacklisting // TODO: need to make a best effort to not re-emit messages if don't have to public class KafkaSpout extends BaseRichSpout { - public static class ZooMeta implements Serializable { - String id; - long offset; - - public ZooMeta() { - - } - - public ZooMeta(String id, long offset) { - this.id = id; - this.offset = offset; - } - } - public static class MessageAndRealOffset { public Message msg; public long offset; @@ -50,14 +35,12 @@ static enum EmitState { public static final Logger LOG = Logger.getLogger(KafkaSpout.class); - - String _uuid = UUID.randomUUID().toString(); SpoutConfig _spoutConfig; SpoutOutputCollector _collector; - TransactionalState _state; PartitionCoordinator _coordinator; DynamicPartitionConnections _connections; + KafkaSpoutState _state; long _lastUpdateMs = 0; @@ -70,31 +53,19 @@ public KafkaSpout(SpoutConfig spoutConf) { @Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { _collector = collector; - Map stateConf = new HashMap(conf); - _connections = new DynamicPartitionConnections(_spoutConfig); - List zkServers = _spoutConfig.zkServers; + List zkServers = _spoutConfig.zkServers; if(zkServers==null) zkServers = (List) conf.get(Config.STORM_ZOOKEEPER_SERVERS); + _state = new KafkaSpoutState(zkServers); - Integer zkPort = _spoutConfig.zkPort; - if(zkPort==null) zkPort = ((Number) conf.get(Config.STORM_ZOOKEEPER_PORT)).intValue(); - - String zkRoot = _spoutConfig.zkRoot; - - stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS, zkServers); - stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_PORT, zkPort); - stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_ROOT, zkRoot); - - Config componentConf = new Config(); - componentConf.registerSerialization(ZooMeta.class); + _connections = new DynamicPartitionConnections(_spoutConfig); // using TransactionalState like this is a hack - _state = TransactionalState.newUserState(stateConf, _spoutConfig.id, componentConf); int totalTasks = context.getComponentTasks(context.getThisComponentId()).size(); if(_spoutConfig.hosts instanceof KafkaConfig.StaticHosts) { - _coordinator = new StaticCoordinator(_connections, _spoutConfig, context.getThisTaskIndex(), totalTasks, _state, _uuid); + _coordinator = new StaticCoordinator(_connections, _spoutConfig, _state, context.getThisTaskIndex(), totalTasks, _uuid); } else { - _coordinator = new ZkCoordinator(_connections, conf, _spoutConfig, context.getThisTaskIndex(), totalTasks, _state, _uuid); + _coordinator = new ZkCoordinator(_connections, conf, _spoutConfig, _state, context.getThisTaskIndex(), totalTasks, _uuid); } } diff --git a/src/jvm/storm/kafka/KafkaSpoutState.java b/src/jvm/storm/kafka/KafkaSpoutState.java new file mode 100644 index 0000000000..92b86df3ed --- /dev/null +++ b/src/jvm/storm/kafka/KafkaSpoutState.java @@ -0,0 +1,29 @@ +package storm.kafka; + +import backtype.storm.Config; +import backtype.storm.utils.Utils; +import java.util.*; +import java.util.Map.Entry; +import org.I0Itec.zkclient.ZkClient; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; + +public class KafkaSpoutState { + ZkClient _zkClient; + + public KafkaSpoutState(List zkServers) { + _zkClient = new ZkClient(Utils.join(zkServers, ","), 10000); + } + + public void writeData(String path, Map data) { + if(!_zkClient.exists(path)) { + _zkClient.createPersistent(path, true); + } + _zkClient.writeData(path, JSONValue.toJSONString(data)); + } + + public Map readData(String path) { + if(!_zkClient.exists(path)) return null; + return (Map)JSONValue.parse((String)_zkClient.readData(path)); + } +} diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index f58d497a47..e9deb90068 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -1,12 +1,8 @@ package storm.kafka; import backtype.storm.spout.SpoutOutputCollector; -import backtype.storm.transactional.state.TransactionalState; import backtype.storm.utils.Utils; -import java.util.LinkedList; -import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; +import java.util.*; import kafka.api.FetchRequest; import kafka.javaapi.consumer.SimpleConsumer; import kafka.javaapi.message.ByteBufferMessageSet; @@ -14,7 +10,6 @@ import org.apache.log4j.Logger; import storm.kafka.KafkaSpout.EmitState; import storm.kafka.KafkaSpout.MessageAndRealOffset; -import storm.kafka.KafkaSpout.ZooMeta; public class PartitionManager { public static final Logger LOG = Logger.getLogger(PartitionManager.class); @@ -33,27 +28,27 @@ public KafkaMessageId(GlobalPartitionId partition, long offset) { SortedSet _pending = new TreeSet(); Long _committedTo; LinkedList _waitingToEmit = new LinkedList(); - TransactionalState _state; GlobalPartitionId _partition; SpoutConfig _spoutConfig; String _topologyInstanceId; SimpleConsumer _consumer; DynamicPartitionConnections _connections; + KafkaSpoutState _state; - public PartitionManager(DynamicPartitionConnections connections, String topologyInstanceId, SpoutConfig spoutConfig, TransactionalState state, GlobalPartitionId id) { - _state = state; + public PartitionManager(DynamicPartitionConnections connections, String topologyInstanceId, KafkaSpoutState state, SpoutConfig spoutConfig, GlobalPartitionId id) { _partition = id; _connections = connections; _spoutConfig = spoutConfig; _topologyInstanceId = topologyInstanceId; - ZooMeta zooMeta = (ZooMeta) _state.getData(committedPath()); - _consumer = connections.register(id.host, id.partition); + _consumer = connections.register(id.host, id.partition); + _state = state; - //the id stuff makes sure the spout doesn't reset the offset if it restarts - if(zooMeta==null || (!topologyInstanceId.equals(zooMeta.id) && spoutConfig.forceFromStart)) { + Map st = _state.readData(committedPath()); + if(st==null || (!topologyInstanceId.equals((String)st.get("id")) && spoutConfig.forceFromStart)) { _committedTo = _consumer.getOffsetsBefore(spoutConfig.topic, id.partition, spoutConfig.startOffsetTime, 1)[0]; } else { - _committedTo = zooMeta.offset; + _committedTo = (Long)st.get("offset"); + LOG.info("Read last commit offset from zookeeper: " + _committedTo); } LOG.info("Starting Kafka " + _consumer.host() + ":" + id.partition + " from offset " + _committedTo); _emittedToOffset = _committedTo; @@ -122,7 +117,12 @@ public void commit() { } if(committedTo!=_committedTo) { LOG.info("Writing committed offset to ZK: " + committedTo); - _state.setData(committedPath(), new ZooMeta(_topologyInstanceId, committedTo)); + + Map data = new LinkedHashMap(); + data.put("id", _topologyInstanceId); + data.put("offset", committedTo); + _state.writeData(committedPath(), data); + LOG.info("Wrote committed offset to ZK: " + committedTo); _committedTo = committedTo; } @@ -130,10 +130,11 @@ public void commit() { } private String committedPath() { - return _spoutConfig.id + "/" + _partition; + return _spoutConfig.zkRoot + "/" + _spoutConfig.id + "/" + _partition; } public void close() { _connections.unregister(_partition.host, _partition.partition); } -} \ No newline at end of file +} + diff --git a/src/jvm/storm/kafka/StaticCoordinator.java b/src/jvm/storm/kafka/StaticCoordinator.java index c8dd9d5945..ddb68e2140 100644 --- a/src/jvm/storm/kafka/StaticCoordinator.java +++ b/src/jvm/storm/kafka/StaticCoordinator.java @@ -1,6 +1,5 @@ package storm.kafka; -import backtype.storm.transactional.state.TransactionalState; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -12,7 +11,7 @@ public class StaticCoordinator implements PartitionCoordinator { Map _managers = new HashMap(); List _allManagers = new ArrayList(); - public StaticCoordinator(DynamicPartitionConnections connections, SpoutConfig config, int taskIndex, int totalTasks, TransactionalState state, String topologyInstanceId) { + public StaticCoordinator(DynamicPartitionConnections connections, SpoutConfig config, KafkaSpoutState state, int taskIndex, int totalTasks, String topologyInstanceId) { StaticHosts hosts = (StaticHosts) config.hosts; List allPartitionIds = new ArrayList(); for(HostPort h: hosts.hosts) { @@ -22,7 +21,7 @@ public StaticCoordinator(DynamicPartitionConnections connections, SpoutConfig co } for(int i=taskIndex; i _managers = new HashMap(); @@ -33,17 +31,17 @@ public class ZkCoordinator implements PartitionCoordinator { int _refreshFreqMs; ZkHosts _brokerConf; DynamicPartitionConnections _connections; + KafkaSpoutState _state; - public ZkCoordinator(DynamicPartitionConnections connections, Map conf, SpoutConfig config, int taskIndex, int totalTasks, TransactionalState state, String topologyInstanceId) { - _config = config; + public ZkCoordinator(DynamicPartitionConnections connections, Map conf, SpoutConfig spoutConfig, KafkaSpoutState state, int taskIndex, int totalTasks, String topologyInstanceId) { + _spoutConfig = spoutConfig; _connections = connections; _taskIndex = taskIndex; _totalTasks = totalTasks; - _state = state; _topologyInstanceId = topologyInstanceId; + _state = state; - - _brokerConf = (ZkHosts) config.hosts; + _brokerConf = (ZkHosts) _spoutConfig.hosts; _refreshFreqMs = _brokerConf.refreshFreqSecs * 1000; try { _curator = CuratorFrameworkFactory.newClient( @@ -56,7 +54,6 @@ public ZkCoordinator(DynamicPartitionConnections connections, Map conf, SpoutCon } catch (IOException e) { throw new RuntimeException(e); } - } @Override @@ -71,7 +68,7 @@ public List getMyManagedPartitions() { void refresh() { try { LOG.info("Refreshing partition manager connections"); - String topicBrokersPath = _brokerConf.brokerZkPath + "/topics/" + _config.topic; + String topicBrokersPath = _brokerConf.brokerZkPath + "/topics/" + _spoutConfig.topic; String brokerInfoPath = _brokerConf.brokerZkPath + "/ids"; List children = _curator.getChildren().forPath(topicBrokersPath); @@ -112,7 +109,7 @@ void refresh() { LOG.info("New partition managers: " + newPartitions.toString()); for(GlobalPartitionId id: newPartitions) { - PartitionManager man = new PartitionManager(_connections, _topologyInstanceId, _config, _state, id); + PartitionManager man = new PartitionManager(_connections, _topologyInstanceId, _state, _spoutConfig, id); _managers.put(id, man); } From ad80c0140fbc9b0310e6828f6c4f89a57be2d311 Mon Sep 17 00:00:00 2001 From: tombrown52 Date: Wed, 25 Jul 2012 10:25:22 -0600 Subject: [PATCH 115/349] Update storm-kafka/src/jvm/storm/kafka/ZkCoordinator.java Fixing a bug that causes partitions to be unassigned if the hash code of the "HostPort" is negative. --- src/jvm/storm/kafka/ZkCoordinator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jvm/storm/kafka/ZkCoordinator.java b/src/jvm/storm/kafka/ZkCoordinator.java index 0405355c04..973b673c4a 100644 --- a/src/jvm/storm/kafka/ZkCoordinator.java +++ b/src/jvm/storm/kafka/ZkCoordinator.java @@ -126,7 +126,7 @@ public PartitionManager getManager(GlobalPartitionId id) { } private boolean myOwnership(GlobalPartitionId id) { - int val = id.host.hashCode() + 23 * id.partition; + int val = Math.abs(id.host.hashCode() + 23 * id.partition); return val % _totalTasks == _taskIndex; } From ef68dd921633b702d9e5d235b5a36d2c50c35070 Mon Sep 17 00:00:00 2001 From: Jason Jackson Date: Tue, 31 Jul 2012 17:51:19 -0700 Subject: [PATCH 116/349] uses netflix curator framework to connect to zookeeper --- project.clj | 2 +- src/jvm/storm/kafka/KafkaSpout.java | 17 ++++- src/jvm/storm/kafka/KafkaSpoutState.java | 29 ------- src/jvm/storm/kafka/PartitionManager.java | 8 +- src/jvm/storm/kafka/StaticCoordinator.java | 2 +- src/jvm/storm/kafka/ZkCoordinator.java | 4 +- src/jvm/storm/kafka/ZkState.java | 89 ++++++++++++++++++++++ 7 files changed, 111 insertions(+), 40 deletions(-) delete mode 100644 src/jvm/storm/kafka/KafkaSpoutState.java create mode 100644 src/jvm/storm/kafka/ZkState.java diff --git a/project.clj b/project.clj index 6ac93e1151..a99a1ddd62 100644 --- a/project.clj +++ b/project.clj @@ -8,7 +8,7 @@ :dependencies [[storm/kafka "0.7.0-incubating" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.7.2"] + :dev-dependencies [[storm "0.7.4"] [org.clojure/clojure "1.4.0"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index d23b32e74d..69d219073f 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -40,7 +40,7 @@ static enum EmitState { SpoutOutputCollector _collector; PartitionCoordinator _coordinator; DynamicPartitionConnections _connections; - KafkaSpoutState _state; + ZkState _state; long _lastUpdateMs = 0; @@ -54,9 +54,15 @@ public KafkaSpout(SpoutConfig spoutConf) { public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { _collector = collector; - List zkServers = _spoutConfig.zkServers; + Map stateConf = new HashMap(conf); + List zkServers = _spoutConfig.zkServers; if(zkServers==null) zkServers = (List) conf.get(Config.STORM_ZOOKEEPER_SERVERS); - _state = new KafkaSpoutState(zkServers); + Integer zkPort = _spoutConfig.zkPort; + if(zkPort==null) zkPort = ((Number) conf.get(Config.STORM_ZOOKEEPER_PORT)).intValue(); + stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS, zkServers); + stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_PORT, zkPort); + stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_ROOT, _spoutConfig.zkRoot); + _state = new ZkState(stateConf); _connections = new DynamicPartitionConnections(_spoutConfig); @@ -70,6 +76,11 @@ public void open(Map conf, TopologyContext context, SpoutOutputCollector collect } + @Override + public void close() { + _state.close(); + } + @Override public void nextTuple() { List managers = _coordinator.getMyManagedPartitions(); diff --git a/src/jvm/storm/kafka/KafkaSpoutState.java b/src/jvm/storm/kafka/KafkaSpoutState.java deleted file mode 100644 index 92b86df3ed..0000000000 --- a/src/jvm/storm/kafka/KafkaSpoutState.java +++ /dev/null @@ -1,29 +0,0 @@ -package storm.kafka; - -import backtype.storm.Config; -import backtype.storm.utils.Utils; -import java.util.*; -import java.util.Map.Entry; -import org.I0Itec.zkclient.ZkClient; -import org.json.simple.JSONObject; -import org.json.simple.JSONValue; - -public class KafkaSpoutState { - ZkClient _zkClient; - - public KafkaSpoutState(List zkServers) { - _zkClient = new ZkClient(Utils.join(zkServers, ","), 10000); - } - - public void writeData(String path, Map data) { - if(!_zkClient.exists(path)) { - _zkClient.createPersistent(path, true); - } - _zkClient.writeData(path, JSONValue.toJSONString(data)); - } - - public Map readData(String path) { - if(!_zkClient.exists(path)) return null; - return (Map)JSONValue.parse((String)_zkClient.readData(path)); - } -} diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index e9deb90068..a8c260b4bf 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -33,9 +33,9 @@ public KafkaMessageId(GlobalPartitionId partition, long offset) { String _topologyInstanceId; SimpleConsumer _consumer; DynamicPartitionConnections _connections; - KafkaSpoutState _state; + ZkState _state; - public PartitionManager(DynamicPartitionConnections connections, String topologyInstanceId, KafkaSpoutState state, SpoutConfig spoutConfig, GlobalPartitionId id) { + public PartitionManager(DynamicPartitionConnections connections, String topologyInstanceId, ZkState state, SpoutConfig spoutConfig, GlobalPartitionId id) { _partition = id; _connections = connections; _spoutConfig = spoutConfig; @@ -43,7 +43,7 @@ public PartitionManager(DynamicPartitionConnections connections, String topology _consumer = connections.register(id.host, id.partition); _state = state; - Map st = _state.readData(committedPath()); + Map st = _state.readJSON(committedPath()); if(st==null || (!topologyInstanceId.equals((String)st.get("id")) && spoutConfig.forceFromStart)) { _committedTo = _consumer.getOffsetsBefore(spoutConfig.topic, id.partition, spoutConfig.startOffsetTime, 1)[0]; } else { @@ -121,7 +121,7 @@ public void commit() { Map data = new LinkedHashMap(); data.put("id", _topologyInstanceId); data.put("offset", committedTo); - _state.writeData(committedPath(), data); + _state.writeJSON(committedPath(), data); LOG.info("Wrote committed offset to ZK: " + committedTo); _committedTo = committedTo; diff --git a/src/jvm/storm/kafka/StaticCoordinator.java b/src/jvm/storm/kafka/StaticCoordinator.java index ddb68e2140..c09d809689 100644 --- a/src/jvm/storm/kafka/StaticCoordinator.java +++ b/src/jvm/storm/kafka/StaticCoordinator.java @@ -11,7 +11,7 @@ public class StaticCoordinator implements PartitionCoordinator { Map _managers = new HashMap(); List _allManagers = new ArrayList(); - public StaticCoordinator(DynamicPartitionConnections connections, SpoutConfig config, KafkaSpoutState state, int taskIndex, int totalTasks, String topologyInstanceId) { + public StaticCoordinator(DynamicPartitionConnections connections, SpoutConfig config, ZkState state, int taskIndex, int totalTasks, String topologyInstanceId) { StaticHosts hosts = (StaticHosts) config.hosts; List allPartitionIds = new ArrayList(); for(HostPort h: hosts.hosts) { diff --git a/src/jvm/storm/kafka/ZkCoordinator.java b/src/jvm/storm/kafka/ZkCoordinator.java index 973b673c4a..189b561b8c 100644 --- a/src/jvm/storm/kafka/ZkCoordinator.java +++ b/src/jvm/storm/kafka/ZkCoordinator.java @@ -31,9 +31,9 @@ public class ZkCoordinator implements PartitionCoordinator { int _refreshFreqMs; ZkHosts _brokerConf; DynamicPartitionConnections _connections; - KafkaSpoutState _state; + ZkState _state; - public ZkCoordinator(DynamicPartitionConnections connections, Map conf, SpoutConfig spoutConfig, KafkaSpoutState state, int taskIndex, int totalTasks, String topologyInstanceId) { + public ZkCoordinator(DynamicPartitionConnections connections, Map conf, SpoutConfig spoutConfig, ZkState state, int taskIndex, int totalTasks, String topologyInstanceId) { _spoutConfig = spoutConfig; _connections = connections; _taskIndex = taskIndex; diff --git a/src/jvm/storm/kafka/ZkState.java b/src/jvm/storm/kafka/ZkState.java new file mode 100644 index 0000000000..6121079fc7 --- /dev/null +++ b/src/jvm/storm/kafka/ZkState.java @@ -0,0 +1,89 @@ +package storm.kafka; + +import backtype.storm.Config; +import backtype.storm.utils.Utils; +import com.netflix.curator.framework.CuratorFramework; +import com.netflix.curator.framework.CuratorFrameworkFactory; +import com.netflix.curator.retry.RetryNTimes; +import java.nio.charset.Charset; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.log4j.Logger; +import org.apache.zookeeper.CreateMode; +import org.json.simple.JSONValue; + +public class ZkState { + public static final Logger LOG = Logger.getLogger(ZkState.class); + CuratorFramework _curator; + + private CuratorFramework newCurator(Map stateConf) throws Exception { + Integer port = (Integer)stateConf.get(Config.TRANSACTIONAL_ZOOKEEPER_PORT); + String serverPorts = ""; + for(String server: (List)stateConf.get(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS)) { + serverPorts = serverPorts + server + ":" + port + ","; + } + return CuratorFrameworkFactory.newClient("localhost:2181,", + Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), + 15000, + new RetryNTimes(Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), + Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); + } + + public ZkState(Map stateConf) { + stateConf = new HashMap(stateConf); + + try { + _curator = newCurator(stateConf); + _curator.start(); + } catch(Exception e) { + throw new RuntimeException(e); + } + } + + public void writeJSON(String path, Map data) { + LOG.info("Writing " + path + " the data " + data.toString()); + writeBytes(path, JSONValue.toJSONString(data).getBytes(Charset.forName("UTF-8"))); + } + + public void writeBytes(String path, byte[] bytes) { + try { + if(_curator.checkExists().forPath(path)==null) { + _curator.create() + .creatingParentsIfNeeded() + .withMode(CreateMode.PERSISTENT) + .forPath(path, bytes); + } else { + _curator.setData().forPath(path, bytes); + } + } catch(Exception e) { + throw new RuntimeException(e); + } + } + + public Map readJSON(String path) { + try { + byte[] b = readBytes(path); + if(b==null) return null; + return (Map)JSONValue.parse(new String(b, "UTF-8")); + } catch(Exception e) { + throw new RuntimeException(e); + } + } + + public byte[] readBytes(String path) { + try { + if(_curator.checkExists().forPath(path)!=null) { + return _curator.getData().forPath(path); + } else { + return null; + } + } catch(Exception e) { + throw new RuntimeException(e); + } + } + + public void close() { + _curator.close(); + } +} From bb542b0bf6244304e835a59f3743df9bc28535fe Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Tue, 31 Jul 2012 22:08:57 -0400 Subject: [PATCH 117/349] merge pull request #8 --- pom.xml | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/pom.xml b/pom.xml index c20a9e9145..a4a2bb31e9 100644 --- a/pom.xml +++ b/pom.xml @@ -1,10 +1,5 @@ - - org.sonatype.oss - oss-parent - 7 - 4.0.0 storm @@ -20,27 +15,14 @@ repo - - scm:git:git@github.com:boneill42/storm-kafka.git - scm:git:git@github.com:boneill42/storm-kafka.git - :git@github.com:boneill42/storm-cassandra.git - - - - - boneill42 - Brian O'Neill - bone@alumni.brown.edu - - clojars.org http://clojars.org/repo - - + + org.apache.kafka From 4a27d30a35b26630fcdd1ea1b0ba84dde3f89fa9 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 1 Aug 2012 14:34:21 -0700 Subject: [PATCH 118/349] fix repos --- project.clj | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/project.clj b/project.clj index 6ac93e1151..ec41604424 100644 --- a/project.clj +++ b/project.clj @@ -1,14 +1,13 @@ -(defproject storm/storm-kafka "0.7.3-dynamic-SNAPSHOT" +(defproject storm/storm-kafka "0.7.3-dynamic-storm8-SNAPSHOT" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} :repositories {"releases" "http://artifactory.local.twitter.com/libs-releases-local" - "snapshots" "http://artifactory.local.twitter.com/libs-snapshots-local" - "artifactory" "http://artifactory.local.twitter.com/repo"} + "snapshots" "http://artifactory.local.twitter.com/libs-snapshots-local"} :dependencies [[storm/kafka "0.7.0-incubating" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.7.2"] + :dev-dependencies [[storm "0.8.0-SNAPSHOT"] [org.clojure/clojure "1.4.0"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) From 1b06f21257c2b3ffff491a55e7d87b29063d8b8b Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 1 Aug 2012 14:35:01 -0700 Subject: [PATCH 119/349] remove pom --- pom.xml | 58 --------------------------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 pom.xml diff --git a/pom.xml b/pom.xml deleted file mode 100644 index a4a2bb31e9..0000000000 --- a/pom.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - 4.0.0 - storm - storm-kafka - 0.72.1-SNAPSHOT - Storm Kafka Support - Storm Kafka Support - - - - Eclipse Public License - v 1.0 - http://www.eclipse.org/legal/epl-v10.html - repo - - - - - - clojars.org - http://clojars.org/repo - - - - - - org.apache.kafka - kafka - 0.7.1 - - - org.scala-lang - scala-library - 2.8.0 - - - storm - storm - 0.7.2 - - - - src/jvm - - - org.apache.maven.plugins - maven-compiler-plugin - - 1.6 - 1.6 - - - - - - - From 8cfa9fe20622d431b16cd29eeb99864042b5e401 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 1 Aug 2012 16:36:45 -0700 Subject: [PATCH 120/349] add trident examples --- project.clj | 3 +- src/jvm/storm/starter/TridentReach.java | 145 ++++++++++++++++++++ src/jvm/storm/starter/TridentWordCount.java | 80 +++++++++++ 3 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 src/jvm/storm/starter/TridentReach.java create mode 100644 src/jvm/storm/starter/TridentWordCount.java diff --git a/project.clj b/project.clj index 98b7f33d51..5d228e9b1e 100644 --- a/project.clj +++ b/project.clj @@ -4,7 +4,6 @@ :javac-options {:debug "true" :fork "true"} :resources-path "multilang" :aot :all - :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] :repositories { "twitter4j" "http://twitter4j.org/maven2" } @@ -14,7 +13,7 @@ [org.twitter4j/twitter4j-stream "2.2.6-SNAPSHOT"] ] - :dev-dependencies [[storm "0.7.4"] + :dev-dependencies [[storm "0.8.0"] [org.clojure/clojure "1.4.0"] ]) diff --git a/src/jvm/storm/starter/TridentReach.java b/src/jvm/storm/starter/TridentReach.java new file mode 100644 index 0000000000..a7c3427001 --- /dev/null +++ b/src/jvm/storm/starter/TridentReach.java @@ -0,0 +1,145 @@ +package storm.starter; + +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.LocalDRPC; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Values; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import storm.trident.TridentState; +import storm.trident.TridentTopology; +import storm.trident.operation.BaseFunction; +import storm.trident.operation.CombinerAggregator; +import storm.trident.operation.TridentCollector; +import storm.trident.operation.builtin.MapGet; +import storm.trident.operation.builtin.Sum; +import storm.trident.state.ReadOnlyState; +import storm.trident.state.State; +import storm.trident.state.StateFactory; +import storm.trident.state.map.ReadOnlyMapState; +import storm.trident.tuple.TridentTuple; + +public class TridentReach { + public static Map> TWEETERS_DB = new HashMap>() {{ + put("foo.com/blog/1", Arrays.asList("sally", "bob", "tim", "george", "nathan")); + put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally", "nathan")); + put("tech.backtype.com/blog/123", Arrays.asList("tim", "mike", "john")); + }}; + + public static Map> FOLLOWERS_DB = new HashMap>() {{ + put("sally", Arrays.asList("bob", "tim", "alice", "adam", "jim", "chris", "jai")); + put("bob", Arrays.asList("sally", "nathan", "jim", "mary", "david", "vivian")); + put("tim", Arrays.asList("alex")); + put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily", "jordan")); + put("adam", Arrays.asList("david", "carissa")); + put("mike", Arrays.asList("john", "bob")); + put("john", Arrays.asList("alice", "nathan", "jim", "mike", "bob")); + }}; + + public static class StaticSingleKeyMapState extends ReadOnlyState implements ReadOnlyMapState { + public static class Factory implements StateFactory { + Map _map; + + public Factory(Map map) { + _map = map; + } + + @Override + public State makeState(Map conf, int partitionIndex, int numPartitions) { + return new StaticSingleKeyMapState(_map); + } + + } + + Map _map; + + public StaticSingleKeyMapState(Map map) { + _map = map; + } + + + @Override + public List multiGet(List> keys) { + List ret = new ArrayList(); + for(List key: keys) { + Object singleKey = key.get(0); + ret.add(_map.get(singleKey)); + } + return ret; + } + + } + + public static class One implements CombinerAggregator { + @Override + public Integer init(TridentTuple tuple) { + return 1; + } + + @Override + public Integer combine(Integer val1, Integer val2) { + return 1; + } + + @Override + public Integer zero() { + return 1; + } + } + + public static class ExpandList extends BaseFunction { + + @Override + public void execute(TridentTuple tuple, TridentCollector collector) { + List l = (List) tuple.getValue(0); + if(l!=null) { + for(Object o: l) { + collector.emit(new Values(o)); + } + } + } + + } + + public static void main(String[] args) throws Exception { + TridentTopology topology = new TridentTopology(); + LocalDRPC drpc = new LocalDRPC(); + + TridentState urlToTweeters = + topology.newStaticState( + new StaticSingleKeyMapState.Factory(TWEETERS_DB)); + TridentState tweetersToFollowers = + topology.newStaticState( + new StaticSingleKeyMapState.Factory(FOLLOWERS_DB)); + + + topology.newDRPCStream("reach", drpc) + .stateQuery(urlToTweeters, new Fields("args"), new MapGet(), new Fields("tweeters")) + .each(new Fields("tweeters"), new ExpandList(), new Fields("tweeter")) + .shuffle() + .stateQuery(tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers")) + .each(new Fields("followers"), new ExpandList(), new Fields("follower")) + .groupBy(new Fields("follower")) + .aggregate(new One(), new Fields("one")) + .aggregate(new Fields("one"), new Sum(), new Fields("reach")); + + Config conf = new Config(); + LocalCluster cluster = new LocalCluster(); + + cluster.submitTopology("reach", conf, topology.build()); + + Thread.sleep(2000); + + System.out.println("REACH: " + drpc.execute("reach", "aaa")); + System.out.println("REACH: " + drpc.execute("reach", "foo.com/blog/1")); + System.out.println("REACH: " + drpc.execute("reach", "engineering.twitter.com/blog/5")); + + + cluster.shutdown(); + drpc.shutdown(); + } +} diff --git a/src/jvm/storm/starter/TridentWordCount.java b/src/jvm/storm/starter/TridentWordCount.java new file mode 100644 index 0000000000..0aea30b49a --- /dev/null +++ b/src/jvm/storm/starter/TridentWordCount.java @@ -0,0 +1,80 @@ +package storm.starter; + +import storm.trident.testing.FixedBatchSpout; +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.LocalDRPC; +import backtype.storm.StormSubmitter; +import backtype.storm.generated.StormTopology; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Values; +import storm.trident.TridentState; +import storm.trident.TridentTopology; +import storm.trident.operation.BaseFunction; +import storm.trident.operation.TridentCollector; +import storm.trident.operation.builtin.Count; +import storm.trident.operation.builtin.FilterNull; +import storm.trident.operation.builtin.MapGet; +import storm.trident.operation.builtin.Sum; +import storm.trident.testing.MemoryMapState; +import storm.trident.tuple.TridentTuple; + + +public class TridentWordCount { + public static class Split extends BaseFunction { + @Override + public void execute(TridentTuple tuple, TridentCollector collector) { + String sentence = tuple.getString(0); + for(String word: sentence.split(" ")) { + collector.emit(new Values(word)); + } + } + } + + public static StormTopology buildTopology(LocalDRPC drpc) { + FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, + new Values("the cow jumped over the moon"), + new Values("the man went to the store and bought some candy"), + new Values("four score and seven years ago"), + new Values("how many apples can you eat"), + new Values("to be or not to be the person")); + spout.setCycle(true); + + TridentTopology topology = new TridentTopology(); + TridentState wordCounts = + topology.newStream("spout1", spout) + .parallelismHint(16) + .each(new Fields("sentence"), new Split(), new Fields("word")) + .groupBy(new Fields("word")) + .persistentAggregate(new MemoryMapState.Factory(), + new Count(), new Fields("count")) + .parallelismHint(16); + + topology.newDRPCStream("words", drpc) + .each(new Fields("args"), new Split(), new Fields("word")) + .groupBy(new Fields("word")) + .stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count")) + .each(new Fields("count"), new FilterNull()) + .aggregate(new Fields("count"), new Sum(), new Fields("sum")) + ; + + return topology.build(); + } + + public static void main(String[] args) throws Exception { + Config conf = new Config(); + conf.setMaxSpoutPending(20); + if(args.length==0) { + LocalDRPC drpc = new LocalDRPC(); + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology("wordCounter", conf, buildTopology(drpc)); + for(int i=0; i<100; i++) { + System.out.println("DRPC RESULT: " + drpc.execute("words", "cat the dog jumped")); + Thread.sleep(1000); + } + } else { + conf.setNumWorkers(3); + StormSubmitter.submitTopology(args[0], conf, buildTopology(null)); + } + } +} From 4b6faeb70e79faaf4f15cd533febc2a8dcf4773f Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Wed, 1 Aug 2012 23:31:57 -0700 Subject: [PATCH 121/349] move trident examples to own package --- src/jvm/storm/starter/{ => trident}/TridentReach.java | 2 +- src/jvm/storm/starter/{ => trident}/TridentWordCount.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/jvm/storm/starter/{ => trident}/TridentReach.java (99%) rename src/jvm/storm/starter/{ => trident}/TridentWordCount.java (99%) diff --git a/src/jvm/storm/starter/TridentReach.java b/src/jvm/storm/starter/trident/TridentReach.java similarity index 99% rename from src/jvm/storm/starter/TridentReach.java rename to src/jvm/storm/starter/trident/TridentReach.java index a7c3427001..c654b2e995 100644 --- a/src/jvm/storm/starter/TridentReach.java +++ b/src/jvm/storm/starter/trident/TridentReach.java @@ -1,4 +1,4 @@ -package storm.starter; +package storm.starter.trident; import backtype.storm.Config; import backtype.storm.LocalCluster; diff --git a/src/jvm/storm/starter/TridentWordCount.java b/src/jvm/storm/starter/trident/TridentWordCount.java similarity index 99% rename from src/jvm/storm/starter/TridentWordCount.java rename to src/jvm/storm/starter/trident/TridentWordCount.java index 0aea30b49a..4712019ae7 100644 --- a/src/jvm/storm/starter/TridentWordCount.java +++ b/src/jvm/storm/starter/trident/TridentWordCount.java @@ -1,4 +1,4 @@ -package storm.starter; +package storm.starter.trident; import storm.trident.testing.FixedBatchSpout; import backtype.storm.Config; From 1b8c94794534e013c8962578c6dccdfd20bea173 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sun, 5 Aug 2012 18:26:49 -0700 Subject: [PATCH 122/349] fix pom --- m2-pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/m2-pom.xml b/m2-pom.xml index b7e34812f3..23b1172478 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -38,7 +38,7 @@ storm storm - 0.7.2 + 0.8.0 provided From a0ca167e17695e39ac907647947737ad258ec7b6 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 6 Aug 2012 13:46:35 -0700 Subject: [PATCH 123/349] turn down kafka logging (don't log when num messages is 0) --- src/jvm/storm/kafka/PartitionManager.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index f58d497a47..23d0c73f95 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -83,20 +83,25 @@ public EmitState next(SpoutOutputCollector collector) { } private void fill() { - LOG.info("Fetching from Kafka: " + _consumer.host() + ":" + _partition.partition + " from offset " + _emittedToOffset); + //LOG.info("Fetching from Kafka: " + _consumer.host() + ":" + _partition.partition + " from offset " + _emittedToOffset); ByteBufferMessageSet msgs = _consumer.fetch( new FetchRequest( _spoutConfig.topic, _partition.partition, _emittedToOffset, _spoutConfig.fetchSizeBytes)); - LOG.info("Fetched " + msgs.underlying().size() + " messages from Kafka: " + _consumer.host() + ":" + _partition.partition); + int numMessages = msgs.underlying().size(); + if(numMessages>0) { + LOG.info("Fetched " + numMessages + " messages from Kafka: " + _consumer.host() + ":" + _partition.partition); + } for(MessageAndOffset msg: msgs) { _pending.add(_emittedToOffset); _waitingToEmit.add(new MessageAndRealOffset(msg.message(), _emittedToOffset)); _emittedToOffset = msg.offset(); } - LOG.info("Added " + msgs.underlying().size() + " messages from Kafka: " + _consumer.host() + ":" + _partition.partition + " to internal buffers"); + if(numMessages>0) { + LOG.info("Added " + numMessages + " messages from Kafka: " + _consumer.host() + ":" + _partition.partition + " to internal buffers"); + } } public void ack(Long offset) { From 4e7868ef2e7d22b3649953d9ae9dc86ae0799246 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Mon, 6 Aug 2012 13:47:25 -0700 Subject: [PATCH 124/349] update version --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index ec41604424..9a90fa8b3b 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.7.3-dynamic-storm8-SNAPSHOT" +(defproject storm/storm-kafka "0.8.0-wip1" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} @@ -7,7 +7,7 @@ :dependencies [[storm/kafka "0.7.0-incubating" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.8.0-SNAPSHOT"] + :dev-dependencies [[storm "0.8.0"] [org.clojure/clojure "1.4.0"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) From 7bfcb121ba2ef957ace4abc4fd552bd3e7ac63b0 Mon Sep 17 00:00:00 2001 From: Argyris Zymnis Date: Mon, 6 Aug 2012 13:49:05 -0700 Subject: [PATCH 125/349] Move to scala 2.9.2 --- project.clj | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/project.clj b/project.clj index 6ac93e1151..c0072f98bd 100644 --- a/project.clj +++ b/project.clj @@ -1,14 +1,14 @@ -(defproject storm/storm-kafka "0.7.3-dynamic-SNAPSHOT" +(defproject storm/storm-kafka "0.7.3-scala292-dynamic-SNAPSHOT" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} - :repositories {"releases" "http://artifactory.local.twitter.com/libs-releases-local" - "snapshots" "http://artifactory.local.twitter.com/libs-snapshots-local" - "artifactory" "http://artifactory.local.twitter.com/repo"} - :dependencies [[storm/kafka "0.7.0-incubating" + ;;:repositories {"releases" "http://artifactory.local.twitter.com/libs-releases-local" + ;; "snapshots" "http://artifactory.local.twitter.com/libs-snapshots-local" + ;; "artifactory" "http://artifactory.local.twitter.com/repo"} + :dependencies [[com.twitter/kafka_2.9.2 "0.7.0" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.7.2"] + :dev-dependencies [[storm "0.8.0-SNAPSHOT"] [org.clojure/clojure "1.4.0"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) From 62f25a73b6621fac8e125c34917d25beb24019a5 Mon Sep 17 00:00:00 2001 From: Argyris Zymnis Date: Tue, 14 Aug 2012 12:29:14 -0700 Subject: [PATCH 126/349] Switch to wip2 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 924d3bb369..01473aace0 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.8.0-wip1-scala292" +(defproject storm/storm-kafka "0.8.0-wip2-scala292" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} From e9688a5f84782450d9c9d5ce0a10adf258f92f1f Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 16 Aug 2012 02:32:21 -0700 Subject: [PATCH 127/349] refactor TridentReach to have a static buildTopology method so it can be used from the REPL --- src/jvm/storm/starter/trident/TridentReach.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/jvm/storm/starter/trident/TridentReach.java b/src/jvm/storm/starter/trident/TridentReach.java index c654b2e995..03da871736 100644 --- a/src/jvm/storm/starter/trident/TridentReach.java +++ b/src/jvm/storm/starter/trident/TridentReach.java @@ -5,6 +5,7 @@ import backtype.storm.LocalDRPC; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Values; +import backtype.storm.generated.StormTopology; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -105,10 +106,8 @@ public void execute(TridentTuple tuple, TridentCollector collector) { } - public static void main(String[] args) throws Exception { + public static StormTopology buildTopology(LocalDRPC drpc) { TridentTopology topology = new TridentTopology(); - LocalDRPC drpc = new LocalDRPC(); - TridentState urlToTweeters = topology.newStaticState( new StaticSingleKeyMapState.Factory(TWEETERS_DB)); @@ -126,11 +125,16 @@ public static void main(String[] args) throws Exception { .groupBy(new Fields("follower")) .aggregate(new One(), new Fields("one")) .aggregate(new Fields("one"), new Sum(), new Fields("reach")); - + return topology.build(); + } + + public static void main(String[] args) throws Exception { + LocalDRPC drpc = new LocalDRPC(); + Config conf = new Config(); LocalCluster cluster = new LocalCluster(); - cluster.submitTopology("reach", conf, topology.build()); + cluster.submitTopology("reach", conf, buildTopology(drpc)); Thread.sleep(2000); From 7ca5f84fcb77833e52a619df4d0ebad509255cce Mon Sep 17 00:00:00 2001 From: Jason Jackson Date: Tue, 21 Aug 2012 21:28:57 -0700 Subject: [PATCH 128/349] x --- project.clj | 2 +- src/jvm/storm/kafka/KafkaSpout.java | 2 +- src/jvm/storm/kafka/PartitionManager.java | 19 ++++++++++++++----- src/jvm/storm/kafka/StaticCoordinator.java | 4 ++-- src/jvm/storm/kafka/ZkCoordinator.java | 14 +++++++------- src/jvm/storm/kafka/ZkState.java | 8 +++++++- 6 files changed, 32 insertions(+), 17 deletions(-) diff --git a/project.clj b/project.clj index a99a1ddd62..33afd6ee3f 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.7.3-dynamic-SNAPSHOT" +(defproject storm/storm-kafka "0.7.4-dynamic-SNAPSHOT" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index 69d219073f..6027cce656 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -69,7 +69,7 @@ public void open(Map conf, TopologyContext context, SpoutOutputCollector collect // using TransactionalState like this is a hack int totalTasks = context.getComponentTasks(context.getThisComponentId()).size(); if(_spoutConfig.hosts instanceof KafkaConfig.StaticHosts) { - _coordinator = new StaticCoordinator(_connections, _spoutConfig, _state, context.getThisTaskIndex(), totalTasks, _uuid); + _coordinator = new StaticCoordinator(_connections, conf, _spoutConfig, _state, context.getThisTaskIndex(), totalTasks, _uuid); } else { _coordinator = new ZkCoordinator(_connections, conf, _spoutConfig, _state, context.getThisTaskIndex(), totalTasks, _uuid); } diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index a8c260b4bf..182a1c529e 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -1,7 +1,9 @@ package storm.kafka; +import backtype.storm.Config; import backtype.storm.spout.SpoutOutputCollector; import backtype.storm.utils.Utils; +import com.google.common.collect.ImmutableMap; import java.util.*; import kafka.api.FetchRequest; import kafka.javaapi.consumer.SimpleConsumer; @@ -34,17 +36,19 @@ public KafkaMessageId(GlobalPartitionId partition, long offset) { SimpleConsumer _consumer; DynamicPartitionConnections _connections; ZkState _state; + Map _stormConf; - public PartitionManager(DynamicPartitionConnections connections, String topologyInstanceId, ZkState state, SpoutConfig spoutConfig, GlobalPartitionId id) { + public PartitionManager(DynamicPartitionConnections connections, String topologyInstanceId, ZkState state, Map stormConf, SpoutConfig spoutConfig, GlobalPartitionId id) { _partition = id; _connections = connections; _spoutConfig = spoutConfig; _topologyInstanceId = topologyInstanceId; _consumer = connections.register(id.host, id.partition); _state = state; + _stormConf = stormConf; Map st = _state.readJSON(committedPath()); - if(st==null || (!topologyInstanceId.equals((String)st.get("id")) && spoutConfig.forceFromStart)) { + if(st==null || (!topologyInstanceId.equals((String)st.get("topology-id")) && spoutConfig.forceFromStart)) { _committedTo = _consumer.getOffsetsBefore(spoutConfig.topic, id.partition, spoutConfig.startOffsetTime, 1)[0]; } else { _committedTo = (Long)st.get("offset"); @@ -118,9 +122,14 @@ public void commit() { if(committedTo!=_committedTo) { LOG.info("Writing committed offset to ZK: " + committedTo); - Map data = new LinkedHashMap(); - data.put("id", _topologyInstanceId); - data.put("offset", committedTo); + Map data = (Map)ImmutableMap.builder() + .put("topology", ImmutableMap.of("id", _topologyInstanceId, + "name", _stormConf.get(Config.TOPOLOGY_NAME))) + .put("offset", committedTo) + .put("partition", _partition.partition) + .put("broker", ImmutableMap.of("host", _partition.host.host, + "port", _partition.host.port)) + .put("topic", _spoutConfig.topic).build(); _state.writeJSON(committedPath(), data); LOG.info("Wrote committed offset to ZK: " + committedTo); diff --git a/src/jvm/storm/kafka/StaticCoordinator.java b/src/jvm/storm/kafka/StaticCoordinator.java index c09d809689..39f2c9ffa6 100644 --- a/src/jvm/storm/kafka/StaticCoordinator.java +++ b/src/jvm/storm/kafka/StaticCoordinator.java @@ -11,7 +11,7 @@ public class StaticCoordinator implements PartitionCoordinator { Map _managers = new HashMap(); List _allManagers = new ArrayList(); - public StaticCoordinator(DynamicPartitionConnections connections, SpoutConfig config, ZkState state, int taskIndex, int totalTasks, String topologyInstanceId) { + public StaticCoordinator(DynamicPartitionConnections connections, Map stormConf, SpoutConfig config, ZkState state, int taskIndex, int totalTasks, String topologyInstanceId) { StaticHosts hosts = (StaticHosts) config.hosts; List allPartitionIds = new ArrayList(); for(HostPort h: hosts.hosts) { @@ -21,7 +21,7 @@ public StaticCoordinator(DynamicPartitionConnections connections, SpoutConfig co } for(int i=taskIndex; i data) { + public void writeJSON(String path, Map data) { LOG.info("Writing " + path + " the data " + data.toString()); writeBytes(path, JSONValue.toJSONString(data).getBytes(Charset.forName("UTF-8"))); } @@ -85,5 +90,6 @@ public byte[] readBytes(String path) { public void close() { _curator.close(); + _curator = null; } } From dfae7ced8b87f3b3d0a90f1388bbc63f6427094a Mon Sep 17 00:00:00 2001 From: Jason Jackson Date: Wed, 22 Aug 2012 14:32:44 -0700 Subject: [PATCH 129/349] refactoring --- project.clj | 2 +- src/jvm/storm/kafka/PartitionManager.java | 2 +- src/jvm/storm/kafka/ZkState.java | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/project.clj b/project.clj index 33afd6ee3f..f80b21a471 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.7.4-dynamic-SNAPSHOT" +(defproject storm/storm-kafka "0.7.4-dynamic-json-SNAPSHOT" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 182a1c529e..052e6551fc 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -47,7 +47,7 @@ public PartitionManager(DynamicPartitionConnections connections, String topology _state = state; _stormConf = stormConf; - Map st = _state.readJSON(committedPath()); + Map st = _state.readJSON(committedPath()); if(st==null || (!topologyInstanceId.equals((String)st.get("topology-id")) && spoutConfig.forceFromStart)) { _committedTo = _consumer.getOffsetsBefore(spoutConfig.topic, id.partition, spoutConfig.startOffsetTime, 1)[0]; } else { diff --git a/src/jvm/storm/kafka/ZkState.java b/src/jvm/storm/kafka/ZkState.java index 6bdb8cd4d8..a72c5d6e6e 100644 --- a/src/jvm/storm/kafka/ZkState.java +++ b/src/jvm/storm/kafka/ZkState.java @@ -23,7 +23,7 @@ private CuratorFramework newCurator(Map stateConf) throws Exception { for(String server: (List)stateConf.get(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS)) { serverPorts = serverPorts + server + ":" + port + ","; } - return CuratorFrameworkFactory.newClient("localhost:2181,", + return CuratorFrameworkFactory.newClient(serverPorts, Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), 15000, new RetryNTimes(Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), @@ -66,11 +66,11 @@ public void writeBytes(String path, byte[] bytes) { } } - public Map readJSON(String path) { + public Map readJSON(String path) { try { byte[] b = readBytes(path); if(b==null) return null; - return (Map)JSONValue.parse(new String(b, "UTF-8")); + return (Map)JSONValue.parse(new String(b, "UTF-8")); } catch(Exception e) { throw new RuntimeException(e); } From 8bf8f1cd22dfae841b9ccf837b852b45ee9f3161 Mon Sep 17 00:00:00 2001 From: Jason Jackson Date: Wed, 22 Aug 2012 16:42:15 -0700 Subject: [PATCH 130/349] clarify logic in case where no ZkState, sets last commit offset to HEAD --- src/jvm/storm/kafka/PartitionManager.java | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 052e6551fc..207b2a3896 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -47,13 +47,30 @@ public PartitionManager(DynamicPartitionConnections connections, String topology _state = state; _stormConf = stormConf; - Map st = _state.readJSON(committedPath()); - if(st==null || (!topologyInstanceId.equals((String)st.get("topology-id")) && spoutConfig.forceFromStart)) { + String jsonTopologyId = null; + Long jsonOffset = null; + try { + Map json = _state.readJSON(committedPath()); + if(json != null) { + jsonTopologyId = (String)((Map)json.get("topology")).get("id"); + jsonOffset = (Long)json.get("offset"); + } + } + catch(Throwable e) { + LOG.warn("Error reading and/or parsing at ZkNode: " + committedPath(), e); + } + + if(jsonTopologyId == null || jsonOffset == null) { // failed to parse JSON? + _committedTo = _consumer.getOffsetsBefore(spoutConfig.topic, id.partition, -1, 1)[0]; + LOG.info("Setting last commit offset to HEAD."); + } else if(!topologyInstanceId.equals(jsonTopologyId) && spoutConfig.forceFromStart) { _committedTo = _consumer.getOffsetsBefore(spoutConfig.topic, id.partition, spoutConfig.startOffsetTime, 1)[0]; + LOG.info("Using startOffsetTime for choose last commit offset."); } else { - _committedTo = (Long)st.get("offset"); + _committedTo = jsonOffset; LOG.info("Read last commit offset from zookeeper: " + _committedTo); } + LOG.info("Starting Kafka " + _consumer.host() + ":" + id.partition + " from offset " + _committedTo); _emittedToOffset = _committedTo; } From a6a0570c09d99b4ff394760963c92395845a77e7 Mon Sep 17 00:00:00 2001 From: Jason Jackson Date: Wed, 22 Aug 2012 16:52:59 -0700 Subject: [PATCH 131/349] reversed last commit offset logic --- src/jvm/storm/kafka/PartitionManager.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 207b2a3896..88b1a4ca2d 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -60,12 +60,12 @@ public PartitionManager(DynamicPartitionConnections connections, String topology LOG.warn("Error reading and/or parsing at ZkNode: " + committedPath(), e); } - if(jsonTopologyId == null || jsonOffset == null) { // failed to parse JSON? + if(!topologyInstanceId.equals(jsonTopologyId) && spoutConfig.forceFromStart) { + _committedTo = _consumer.getOffsetsBefore(spoutConfig.topic, id.partition, spoutConfig.startOffsetTime, 1)[0]; + LOG.info("Using startOffsetTime to choose last commit offset."); + } else if(jsonTopologyId == null || jsonOffset == null) { // failed to parse JSON? _committedTo = _consumer.getOffsetsBefore(spoutConfig.topic, id.partition, -1, 1)[0]; LOG.info("Setting last commit offset to HEAD."); - } else if(!topologyInstanceId.equals(jsonTopologyId) && spoutConfig.forceFromStart) { - _committedTo = _consumer.getOffsetsBefore(spoutConfig.topic, id.partition, spoutConfig.startOffsetTime, 1)[0]; - LOG.info("Using startOffsetTime for choose last commit offset."); } else { _committedTo = jsonOffset; LOG.info("Read last commit offset from zookeeper: " + _committedTo); From 99c7bcf801abf38fcc1df629c2700dbad5e04e35 Mon Sep 17 00:00:00 2001 From: Jason Jackson Date: Thu, 23 Aug 2012 14:20:26 -0700 Subject: [PATCH 132/349] removed SNAPSHOT from project --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index f80b21a471..c8b15e337c 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.7.4-dynamic-json-SNAPSHOT" +(defproject storm/storm-kafka "0.7.5-dynamic-json" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} From 2840a00be2fb4479e481d097fc4aeffc7cffa817 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 6 Sep 2012 15:50:05 -0700 Subject: [PATCH 133/349] bump to 0.8.1 --- m2-pom.xml | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/m2-pom.xml b/m2-pom.xml index 23b1172478..443e1a6872 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -38,7 +38,7 @@ storm storm - 0.8.0 + 0.8.1 provided diff --git a/project.clj b/project.clj index 5d228e9b1e..a01b145e0f 100644 --- a/project.clj +++ b/project.clj @@ -13,7 +13,7 @@ [org.twitter4j/twitter4j-stream "2.2.6-SNAPSHOT"] ] - :dev-dependencies [[storm "0.8.0"] + :dev-dependencies [[storm "0.8.1"] [org.clojure/clojure "1.4.0"] ]) From acb39026f599f5a0e29701d6232de8fd5a3332bf Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Tue, 11 Sep 2012 14:43:04 -0700 Subject: [PATCH 134/349] remove twitter4j as a dependency and comment out twitter4j examples since twitter4j repo frequently has issues --- project.clj | 6 +++--- src/jvm/storm/starter/PrintSampleStream.java | 5 +++++ src/jvm/storm/starter/spout/TwitterSampleSpout.java | 3 +++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index a01b145e0f..1415ffb5cb 100644 --- a/project.clj +++ b/project.clj @@ -5,12 +5,12 @@ :resources-path "multilang" :aot :all :repositories { - "twitter4j" "http://twitter4j.org/maven2" +;; "twitter4j" "http://twitter4j.org/maven2" } :dependencies [ - [org.twitter4j/twitter4j-core "2.2.6-SNAPSHOT"] - [org.twitter4j/twitter4j-stream "2.2.6-SNAPSHOT"] +;; [org.twitter4j/twitter4j-core "2.2.6-SNAPSHOT"] +;; [org.twitter4j/twitter4j-stream "2.2.6-SNAPSHOT"] ] :dev-dependencies [[storm "0.8.1"] diff --git a/src/jvm/storm/starter/PrintSampleStream.java b/src/jvm/storm/starter/PrintSampleStream.java index b3524b58d6..e880e16229 100644 --- a/src/jvm/storm/starter/PrintSampleStream.java +++ b/src/jvm/storm/starter/PrintSampleStream.java @@ -1,3 +1,7 @@ +/* +// to use this example, uncomment the twitter4j dependency information in the project.clj, +// uncomment storm.starter.spout.TwitterSampleSpout, and uncomment this class + package storm.starter; import storm.starter.spout.TwitterSampleSpout; @@ -30,3 +34,4 @@ public static void main(String[] args) { cluster.shutdown(); } } +*/ \ No newline at end of file diff --git a/src/jvm/storm/starter/spout/TwitterSampleSpout.java b/src/jvm/storm/starter/spout/TwitterSampleSpout.java index 4c0e1332b0..0a48cf2986 100644 --- a/src/jvm/storm/starter/spout/TwitterSampleSpout.java +++ b/src/jvm/storm/starter/spout/TwitterSampleSpout.java @@ -1,3 +1,5 @@ +/* + package storm.starter.spout; import backtype.storm.Config; @@ -100,3 +102,4 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } } +*/ \ No newline at end of file From 92b6a0aa112e10dc7059e40940ef7af6d696f308 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 13 Oct 2012 10:04:58 -0700 Subject: [PATCH 135/349] refactored reading kafka broker info from zookeeper into its own class --- src/jvm/storm/kafka/DynamicBrokersReader.java | 95 +++++++++++++++++++ src/jvm/storm/kafka/ZkCoordinator.java | 81 ++++------------ 2 files changed, 111 insertions(+), 65 deletions(-) create mode 100644 src/jvm/storm/kafka/DynamicBrokersReader.java diff --git a/src/jvm/storm/kafka/DynamicBrokersReader.java b/src/jvm/storm/kafka/DynamicBrokersReader.java new file mode 100644 index 0000000000..30325b152d --- /dev/null +++ b/src/jvm/storm/kafka/DynamicBrokersReader.java @@ -0,0 +1,95 @@ +package storm.kafka; + +import backtype.storm.Config; +import backtype.storm.utils.Utils; +import com.netflix.curator.framework.CuratorFramework; +import com.netflix.curator.framework.CuratorFrameworkFactory; +import com.netflix.curator.retry.RetryNTimes; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DynamicBrokersReader { + + CuratorFramework _curator; + String _zkPath; + String _topic; + + public DynamicBrokersReader(Map conf, String zkStr, String zkPath, String topic) { + try { + _zkPath = zkPath; + _topic = topic; + _curator = CuratorFrameworkFactory.newClient( + zkStr, + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), + 15000, + new RetryNTimes(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); + _curator.start(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Map of host to List of port and number of partitions. + * + * {"host1.mycompany.com" -> [9092, 5]} + */ + public Map getBrokerInfo() { + Map ret = new HashMap(); + try { + String topicBrokersPath = _zkPath + "/topics/" + _topic; + String brokerInfoPath = _zkPath + "/ids"; + List children = _curator.getChildren().forPath(topicBrokersPath); + + for(String c: children) { + try { + byte[] numPartitionsData = _curator.getData().forPath(topicBrokersPath + "/" + c); + byte[] hostPortData = _curator.getData().forPath(brokerInfoPath + "/" + c); + + + + HostPort hp = getBrokerHost(hostPortData); + int numPartitions = getNumPartitions(numPartitionsData); + List info = new ArrayList(); + info.add((long)hp.port); + info.add((long)numPartitions); + ret.put(hp.host, info); + + } catch(org.apache.zookeeper.KeeperException.NoNodeException e) { + + } + } + } catch(Exception e) { + throw new RuntimeException(e); + } + return ret; + } + + public void close() { + _curator.close(); + } + + private static HostPort getBrokerHost(byte[] contents) { + try { + String[] hostString = new String(contents, "UTF-8").split(":"); + String host = hostString[hostString.length - 2]; + int port = Integer.parseInt(hostString[hostString.length - 1]); + return new HostPort(host, port); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + private static int getNumPartitions(byte[] contents) { + try { + return Integer.parseInt(new String(contents, "UTF-8")); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/jvm/storm/kafka/ZkCoordinator.java b/src/jvm/storm/kafka/ZkCoordinator.java index f287d8070b..6d894b9f8e 100644 --- a/src/jvm/storm/kafka/ZkCoordinator.java +++ b/src/jvm/storm/kafka/ZkCoordinator.java @@ -1,20 +1,12 @@ package storm.kafka; -import backtype.storm.Config; import backtype.storm.transactional.state.TransactionalState; -import backtype.storm.utils.Utils; -import com.netflix.curator.framework.CuratorFramework; -import com.netflix.curator.framework.CuratorFrameworkFactory; -import com.netflix.curator.retry.RetryNTimes; -import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import kafka.javaapi.consumer.SimpleConsumer; import org.apache.log4j.Logger; import storm.kafka.KafkaConfig.ZkHosts; @@ -26,13 +18,12 @@ public class ZkCoordinator implements PartitionCoordinator { int _totalTasks; TransactionalState _state; String _topologyInstanceId; - CuratorFramework _curator; Map _managers = new HashMap(); List _cachedList; Long _lastRefreshTime = null; int _refreshFreqMs; - ZkHosts _brokerConf; DynamicPartitionConnections _connections; + DynamicBrokersReader _reader; public ZkCoordinator(DynamicPartitionConnections connections, Map conf, SpoutConfig config, int taskIndex, int totalTasks, TransactionalState state, String topologyInstanceId) { _config = config; @@ -43,19 +34,9 @@ public ZkCoordinator(DynamicPartitionConnections connections, Map conf, SpoutCon _topologyInstanceId = topologyInstanceId; - _brokerConf = (ZkHosts) config.hosts; - _refreshFreqMs = _brokerConf.refreshFreqSecs * 1000; - try { - _curator = CuratorFrameworkFactory.newClient( - _brokerConf.brokerZkStr, - Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), - 15000, - new RetryNTimes(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), - Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); - _curator.start(); - } catch (IOException e) { - throw new RuntimeException(e); - } + ZkHosts brokerConf = (ZkHosts) config.hosts; + _refreshFreqMs = brokerConf.refreshFreqSecs * 1000; + _reader = new DynamicBrokersReader(conf, brokerConf.brokerZkStr, brokerConf.brokerZkPath, config.topic); } @@ -71,31 +52,22 @@ public List getMyManagedPartitions() { void refresh() { try { LOG.info("Refreshing partition manager connections"); - String topicBrokersPath = _brokerConf.brokerZkPath + "/topics/" + _config.topic; - String brokerInfoPath = _brokerConf.brokerZkPath + "/ids"; - List children = _curator.getChildren().forPath(topicBrokersPath); + Map brokerInfo = _reader.getBrokerInfo(); Set mine = new HashSet(); - for(String c: children) { - try { - byte[] numPartitionsData = _curator.getData().forPath(topicBrokersPath + "/" + c); - byte[] hostPortData = _curator.getData().forPath(brokerInfoPath + "/" + c); - - - - HostPort hp = getBrokerHost(hostPortData); - int numPartitions = getNumPartitions(numPartitionsData); - - for(int i=0; i curr = _managers.keySet(); Set newPartitions = new HashSet(mine); newPartitions.removeAll(curr); @@ -131,26 +103,5 @@ public PartitionManager getManager(GlobalPartitionId id) { private boolean myOwnership(GlobalPartitionId id) { int val = Math.abs(id.host.hashCode() + 23 * id.partition); return val % _totalTasks == _taskIndex; - } - - - - private static HostPort getBrokerHost(byte[] contents) { - try { - String[] hostString = new String(contents, "UTF-8").split(":"); - String host = hostString[hostString.length - 2]; - int port = Integer.parseInt(hostString[hostString.length - 1]); - return new HostPort(host, port); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - } - - private static int getNumPartitions(byte[] contents) { - try { - return Integer.parseInt(new String(contents, "UTF-8")); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - } + } } From f046c2bf8a297d237e076868655e70606860de59 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 13 Oct 2012 10:05:36 -0700 Subject: [PATCH 136/349] remove transactional topology kafkaspout implementations --- src/jvm/storm/kafka/BatchMeta.java | 6 - src/jvm/storm/kafka/FailedFetchException.java | 7 -- src/jvm/storm/kafka/KafkaUtils.java | 53 -------- .../kafka/OpaqueTransactionalKafkaSpout.java | 100 --------------- src/jvm/storm/kafka/TestTopology.java | 28 ++--- .../storm/kafka/TransactionalKafkaSpout.java | 114 ------------------ 6 files changed, 14 insertions(+), 294 deletions(-) delete mode 100644 src/jvm/storm/kafka/BatchMeta.java delete mode 100644 src/jvm/storm/kafka/FailedFetchException.java delete mode 100644 src/jvm/storm/kafka/KafkaUtils.java delete mode 100644 src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java delete mode 100644 src/jvm/storm/kafka/TransactionalKafkaSpout.java diff --git a/src/jvm/storm/kafka/BatchMeta.java b/src/jvm/storm/kafka/BatchMeta.java deleted file mode 100644 index 4afbe8aeec..0000000000 --- a/src/jvm/storm/kafka/BatchMeta.java +++ /dev/null @@ -1,6 +0,0 @@ -package storm.kafka; - -public class BatchMeta { - long offset; - long nextOffset; -} diff --git a/src/jvm/storm/kafka/FailedFetchException.java b/src/jvm/storm/kafka/FailedFetchException.java deleted file mode 100644 index 0a3b4b8a57..0000000000 --- a/src/jvm/storm/kafka/FailedFetchException.java +++ /dev/null @@ -1,7 +0,0 @@ -package storm.kafka; - -public class FailedFetchException extends RuntimeException { - public FailedFetchException(Exception e) { - super(e); - } -} diff --git a/src/jvm/storm/kafka/KafkaUtils.java b/src/jvm/storm/kafka/KafkaUtils.java deleted file mode 100644 index e0f4151f61..0000000000 --- a/src/jvm/storm/kafka/KafkaUtils.java +++ /dev/null @@ -1,53 +0,0 @@ -package storm.kafka; - -import backtype.storm.coordination.BatchOutputCollector; -import backtype.storm.transactional.TransactionAttempt; -import backtype.storm.utils.Utils; -import java.net.ConnectException; -import java.util.ArrayList; -import java.util.List; -import kafka.api.FetchRequest; -import kafka.javaapi.consumer.SimpleConsumer; -import kafka.javaapi.message.ByteBufferMessageSet; -import kafka.message.Message; -import kafka.message.MessageAndOffset; -import storm.kafka.KafkaConfig.StaticHosts; - -public class KafkaUtils { - - - public static BatchMeta emitPartitionBatchNew(KafkaConfig config, int partition, SimpleConsumer consumer, TransactionAttempt attempt, BatchOutputCollector collector, BatchMeta lastMeta) { - StaticHosts hosts = (StaticHosts) config.hosts; - long offset = 0; - if(lastMeta!=null) { - offset = lastMeta.nextOffset; - } - ByteBufferMessageSet msgs; - try { - msgs = consumer.fetch(new FetchRequest(config.topic, partition % hosts.partitionsPerHost, offset, config.fetchSizeBytes)); - } catch(Exception e) { - if(e instanceof ConnectException) { - throw new FailedFetchException(e); - } else { - throw new RuntimeException(e); - } - } - long endoffset = offset; - for(MessageAndOffset msg: msgs) { - emit(config, attempt, collector, msg.message()); - endoffset = msg.offset(); - } - BatchMeta newMeta = new BatchMeta(); - newMeta.offset = offset; - newMeta.nextOffset = endoffset; - return newMeta; - } - - public static void emit(KafkaConfig config, TransactionAttempt attempt, BatchOutputCollector collector, Message msg) { - List values = config.scheme.deserialize(Utils.toByteArray(msg.payload())); - List toEmit = new ArrayList(); - toEmit.add(attempt); - toEmit.addAll(values); - collector.emit(toEmit); - } -} diff --git a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java b/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java deleted file mode 100644 index cc78cf5257..0000000000 --- a/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java +++ /dev/null @@ -1,100 +0,0 @@ -package storm.kafka; - -import backtype.storm.coordination.BatchOutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.transactional.TransactionAttempt; -import backtype.storm.transactional.partitioned.IOpaquePartitionedTransactionalSpout; -import backtype.storm.tuple.Fields; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import kafka.javaapi.consumer.SimpleConsumer; -import org.apache.log4j.Logger; -import storm.kafka.KafkaConfig.StaticHosts; - - -public class OpaqueTransactionalKafkaSpout implements IOpaquePartitionedTransactionalSpout { - public static final Logger LOG = Logger.getLogger(OpaqueTransactionalKafkaSpout.class); - - public static final String ATTEMPT_FIELD = OpaqueTransactionalKafkaSpout.class.getCanonicalName() + "/attempt"; - - KafkaConfig _config; - - public OpaqueTransactionalKafkaSpout(KafkaConfig config) { - _config = config; - } - - @Override - public IOpaquePartitionedTransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) { - return new Emitter(); - } - - @Override - public IOpaquePartitionedTransactionalSpout.Coordinator getCoordinator(Map map, TopologyContext tc) { - return new Coordinator(); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - List fields = new ArrayList(_config.scheme.getOutputFields().toList()); - fields.add(0, ATTEMPT_FIELD); - declarer.declare(new Fields(fields)); - } - - @Override - public Map getComponentConfiguration() { - backtype.storm.Config conf = new backtype.storm.Config(); - conf.registerSerialization(BatchMeta.class); - return conf; - } - - class Coordinator implements IOpaquePartitionedTransactionalSpout.Coordinator { - @Override - public boolean isReady() { - //TODO: can do a more sophisticated strategy by looking at the high water marks for each partition - return true; - } - - @Override - public void close() { - } - } - - class Emitter implements IOpaquePartitionedTransactionalSpout.Emitter { - StaticPartitionConnections _connections; - - public Emitter() { - _connections = new StaticPartitionConnections(_config); - } - - @Override - public BatchMeta emitPartitionBatch(TransactionAttempt attempt, BatchOutputCollector collector, int partition, BatchMeta lastMeta) { - try { - SimpleConsumer consumer = _connections.getConsumer(partition); - return KafkaUtils.emitPartitionBatchNew(_config, partition, consumer, attempt, collector, lastMeta); - } catch(FailedFetchException e) { - LOG.warn("Failed to fetch from partition " + partition); - if(lastMeta==null) { - return null; - } else { - BatchMeta ret = new BatchMeta(); - ret.offset = lastMeta.nextOffset; - ret.nextOffset = lastMeta.nextOffset; - return ret; - } - } - } - - @Override - public int numPartitions() { - StaticHosts hosts = (StaticHosts) _config.hosts; - return hosts.hosts.size() * hosts.partitionsPerHost; - } - - @Override - public void close() { - _connections.close(); - } - } -} diff --git a/src/jvm/storm/kafka/TestTopology.java b/src/jvm/storm/kafka/TestTopology.java index 74bbc8fd3e..4fa6504d29 100644 --- a/src/jvm/storm/kafka/TestTopology.java +++ b/src/jvm/storm/kafka/TestTopology.java @@ -25,19 +25,19 @@ public void execute(Tuple tuple, BasicOutputCollector collector) { } public static void main(String [] args) throws Exception { - List hosts = new ArrayList(); - hosts.add("localhost"); - KafkaConfig kafkaConf = new KafkaConfig(StaticHosts.fromHostString(hosts, 3), "test"); - kafkaConf.scheme = new StringScheme(); - LocalCluster cluster = new LocalCluster(); - TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("id", "spout", - new OpaqueTransactionalKafkaSpout(kafkaConf), 1); - builder.setBolt("printer", new PrinterBolt()) - .shuffleGrouping("spout"); - Config config = new Config(); - - cluster.submitTopology("kafka-test", config, builder.buildTopology()); - - Thread.sleep(600000); +// List hosts = new ArrayList(); +// hosts.add("localhost"); +// KafkaConfig kafkaConf = new KafkaConfig(StaticHosts.fromHostString(hosts, 3), "test"); +// kafkaConf.scheme = new StringScheme(); +// LocalCluster cluster = new LocalCluster(); +// TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("id", "spout", +// new OpaqueTransactionalKafkaSpout(kafkaConf), 1); +// builder.setBolt("printer", new PrinterBolt()) +// .shuffleGrouping("spout"); +// Config config = new Config(); +// +// cluster.submitTopology("kafka-test", config, builder.buildTopology()); +// +// Thread.sleep(600000); } } diff --git a/src/jvm/storm/kafka/TransactionalKafkaSpout.java b/src/jvm/storm/kafka/TransactionalKafkaSpout.java deleted file mode 100644 index d5b6e10412..0000000000 --- a/src/jvm/storm/kafka/TransactionalKafkaSpout.java +++ /dev/null @@ -1,114 +0,0 @@ -package storm.kafka; - -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BasePartitionedTransactionalSpout; -import backtype.storm.transactional.TransactionAttempt; -import backtype.storm.transactional.partitioned.IPartitionedTransactionalSpout; -import backtype.storm.coordination.BatchOutputCollector; -import backtype.storm.tuple.Fields; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import kafka.api.FetchRequest; -import kafka.javaapi.consumer.SimpleConsumer; -import kafka.javaapi.message.ByteBufferMessageSet; -import kafka.message.MessageAndOffset; -import storm.kafka.KafkaConfig.StaticHosts; - - -public class TransactionalKafkaSpout extends BasePartitionedTransactionalSpout { - public static final String ATTEMPT_FIELD = TransactionalKafkaSpout.class.getCanonicalName() + "/attempt"; - - KafkaConfig _config; - - public TransactionalKafkaSpout(KafkaConfig config) { - _config = config; - } - - class Coordinator implements IPartitionedTransactionalSpout.Coordinator { - @Override - public int numPartitions() { - return computeNumPartitions(); - } - - @Override - public void close() { - } - - @Override - public boolean isReady() { - //TODO: can do a more sophisticated strategy by looking at the high water marks for each partition - return true; - } - } - - class Emitter implements IPartitionedTransactionalSpout.Emitter { - StaticPartitionConnections _connections; - int partitionsPerHost; - - public Emitter() { - _connections = new StaticPartitionConnections(_config); - StaticHosts hosts = (StaticHosts) _config.hosts; - partitionsPerHost = hosts.partitionsPerHost; - } - - @Override - public BatchMeta emitPartitionBatchNew(TransactionAttempt attempt, BatchOutputCollector collector, int partition, BatchMeta lastMeta) { - SimpleConsumer consumer = _connections.getConsumer(partition); - - return KafkaUtils.emitPartitionBatchNew(_config, partition, consumer, attempt, collector, lastMeta); - } - - @Override - public void emitPartitionBatch(TransactionAttempt attempt, BatchOutputCollector collector, int partition, BatchMeta meta) { - SimpleConsumer consumer = _connections.getConsumer(partition); - - ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition % partitionsPerHost, meta.offset, _config.fetchSizeBytes)); - long offset = meta.offset; - for(MessageAndOffset msg: msgs) { - if(offset == meta.nextOffset) break; - if(offset > meta.nextOffset) { - throw new RuntimeException("Error when re-emitting batch. overshot the end offset"); - } - KafkaUtils.emit(_config, attempt, collector, msg.message()); - offset = msg.offset(); - } - } - - @Override - public void close() { - _connections.close(); - } - } - - - @Override - public IPartitionedTransactionalSpout.Coordinator getCoordinator(Map conf, TopologyContext context) { - return new Coordinator(); - } - - @Override - public IPartitionedTransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) { - return new Emitter(); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - List fields = new ArrayList(_config.scheme.getOutputFields().toList()); - fields.add(0, ATTEMPT_FIELD); - declarer.declare(new Fields(fields)); - } - - private int computeNumPartitions() { - StaticHosts hosts = (StaticHosts) _config.hosts; - return hosts.hosts.size() * hosts.partitionsPerHost; - } - - @Override - public Map getComponentConfiguration() { - backtype.storm.Config conf = new backtype.storm.Config(); - conf.registerSerialization(BatchMeta.class); - return conf; - } -} \ No newline at end of file From 32076c24839b1ea3763fc63918aeeb2897cf6b77 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 13 Oct 2012 13:10:39 -0700 Subject: [PATCH 137/349] moved trident-kafka into storm-kafka --- src/jvm/storm/kafka/KafkaConfig.java | 2 +- .../kafka/trident/DefaultCoordinator.java | 14 +++ .../kafka/trident/FailedFetchException.java | 7 ++ .../kafka/trident/IBatchCoordinator.java | 8 ++ src/jvm/storm/kafka/trident/KafkaUtils.java | 62 ++++++++++ .../trident/OpaqueTridentKafkaSpout.java | 95 +++++++++++++++ .../TransactionalTridentKafkaSpout.java | 112 ++++++++++++++++++ .../kafka/trident/TridentKafkaConfig.java | 12 ++ 8 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 src/jvm/storm/kafka/trident/DefaultCoordinator.java create mode 100644 src/jvm/storm/kafka/trident/FailedFetchException.java create mode 100644 src/jvm/storm/kafka/trident/IBatchCoordinator.java create mode 100644 src/jvm/storm/kafka/trident/KafkaUtils.java create mode 100644 src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java create mode 100644 src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java create mode 100644 src/jvm/storm/kafka/trident/TridentKafkaConfig.java diff --git a/src/jvm/storm/kafka/KafkaConfig.java b/src/jvm/storm/kafka/KafkaConfig.java index 81ccc1df28..943cef1acd 100644 --- a/src/jvm/storm/kafka/KafkaConfig.java +++ b/src/jvm/storm/kafka/KafkaConfig.java @@ -45,7 +45,7 @@ public ZkHosts(String brokerZkStr, String brokerZkPath) { } - BrokerHosts hosts; + public BrokerHosts hosts; public int fetchSizeBytes = 1024*1024; public int socketTimeoutMs = 10000; public int bufferSizeBytes = 1024*1024; diff --git a/src/jvm/storm/kafka/trident/DefaultCoordinator.java b/src/jvm/storm/kafka/trident/DefaultCoordinator.java new file mode 100644 index 0000000000..3a477062cf --- /dev/null +++ b/src/jvm/storm/kafka/trident/DefaultCoordinator.java @@ -0,0 +1,14 @@ +package storm.kafka.trident; + +public class DefaultCoordinator implements IBatchCoordinator { + + @Override + public boolean isReady(long txid) { + return true; + } + + @Override + public void close() { + } + +} diff --git a/src/jvm/storm/kafka/trident/FailedFetchException.java b/src/jvm/storm/kafka/trident/FailedFetchException.java new file mode 100644 index 0000000000..c4fcc6181d --- /dev/null +++ b/src/jvm/storm/kafka/trident/FailedFetchException.java @@ -0,0 +1,7 @@ +package storm.kafka.trident; + +public class FailedFetchException extends RuntimeException { + public FailedFetchException(Exception e) { + super(e); + } +} diff --git a/src/jvm/storm/kafka/trident/IBatchCoordinator.java b/src/jvm/storm/kafka/trident/IBatchCoordinator.java new file mode 100644 index 0000000000..9199a8da1b --- /dev/null +++ b/src/jvm/storm/kafka/trident/IBatchCoordinator.java @@ -0,0 +1,8 @@ +package storm.kafka.trident; + +import java.io.Serializable; + +public interface IBatchCoordinator extends Serializable { + boolean isReady(long txid); + void close(); +} diff --git a/src/jvm/storm/kafka/trident/KafkaUtils.java b/src/jvm/storm/kafka/trident/KafkaUtils.java new file mode 100644 index 0000000000..b91f4ccffa --- /dev/null +++ b/src/jvm/storm/kafka/trident/KafkaUtils.java @@ -0,0 +1,62 @@ +package storm.kafka.trident; + +import backtype.storm.utils.Utils; +import java.net.ConnectException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import kafka.api.FetchRequest; +import kafka.javaapi.consumer.SimpleConsumer; +import kafka.javaapi.message.ByteBufferMessageSet; +import kafka.message.Message; +import kafka.message.MessageAndOffset; +import storm.kafka.KafkaConfig.StaticHosts; +import storm.trident.operation.TridentCollector; +import storm.trident.topology.TransactionAttempt; + +public class KafkaUtils { + + + public static Map emitPartitionBatchNew(TridentKafkaConfig config, int partition, SimpleConsumer consumer, TransactionAttempt attempt, TridentCollector collector, Map lastMeta, String topologyInstanceId) { + StaticHosts hosts = (StaticHosts) config.hosts; + long offset; + if(lastMeta!=null) { + if(config.forceFromStart && !topologyInstanceId.equals(lastMeta.get("instanceId"))) { + offset = consumer.getOffsetsBefore(config.topic, partition % hosts.partitionsPerHost, config.startOffsetTime, 1)[0]; + } else { + offset = (Long) lastMeta.get("nextOffset"); + } + } else { + long startTime = -1; + if(config.forceFromStart) startTime = config.startOffsetTime; + offset = consumer.getOffsetsBefore(config.topic, partition % hosts.partitionsPerHost, startTime, 1)[0]; + } + ByteBufferMessageSet msgs; + try { + msgs = consumer.fetch(new FetchRequest(config.topic, partition % hosts.partitionsPerHost, offset, config.fetchSizeBytes)); + } catch(Exception e) { + if(e instanceof ConnectException) { + throw new FailedFetchException(e); + } else { + throw new RuntimeException(e); + } + } + long endoffset = offset; + for(MessageAndOffset msg: msgs) { + emit(config, attempt, collector, msg.message()); + endoffset = msg.offset(); + } + Map newMeta = new HashMap(); + newMeta.put("offset", offset); + newMeta.put("nextOffset", endoffset); + newMeta.put("instanceId", topologyInstanceId); + return newMeta; + } + + public static void emit(TridentKafkaConfig config, TransactionAttempt attempt, TridentCollector collector, Message msg) { + List values = config.scheme.deserialize(Utils.toByteArray(msg.payload())); + if(values!=null) { + collector.emit(values); + } + } +} diff --git a/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java b/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java new file mode 100644 index 0000000000..d31fdcf759 --- /dev/null +++ b/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java @@ -0,0 +1,95 @@ +package storm.kafka.trident; + +import backtype.storm.task.TopologyContext; +import backtype.storm.tuple.Fields; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import kafka.javaapi.consumer.SimpleConsumer; +import org.apache.log4j.Logger; +import storm.kafka.KafkaConfig.StaticHosts; +import storm.kafka.StaticPartitionConnections; +import storm.trident.operation.TridentCollector; +import storm.trident.spout.IOpaquePartitionedTridentSpout; +import storm.trident.topology.TransactionAttempt; + + +public class OpaqueTridentKafkaSpout implements IOpaquePartitionedTridentSpout { + public static final Logger LOG = Logger.getLogger(OpaqueTridentKafkaSpout.class); + + TridentKafkaConfig _config; + String _topologyInstanceId = UUID.randomUUID().toString(); + + public OpaqueTridentKafkaSpout(TridentKafkaConfig config) { + _config = config; + } + + @Override + public IOpaquePartitionedTridentSpout.Emitter getEmitter(Map conf, TopologyContext context) { + return new Emitter(); + } + + @Override + public IOpaquePartitionedTridentSpout.Coordinator getCoordinator(Map map, TopologyContext tc) { + return new Coordinator(); + } + + @Override + public Fields getOutputFields() { + return _config.scheme.getOutputFields(); + } + + @Override + public Map getComponentConfiguration() { + return null; + } + + class Coordinator implements IOpaquePartitionedTridentSpout.Coordinator { + @Override + public void close() { + _config.coordinator.close(); + } + + @Override + public boolean isReady(long txid) { + return _config.coordinator.isReady(txid); + } + } + + class Emitter implements IOpaquePartitionedTridentSpout.Emitter { + StaticPartitionConnections _connections; + + public Emitter() { + _connections = new StaticPartitionConnections(_config); + } + + @Override + public Map emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, int partition, Map lastMeta) { + try { + SimpleConsumer consumer = _connections.getConsumer(partition); + return KafkaUtils.emitPartitionBatchNew(_config, partition, consumer, attempt, collector, lastMeta, _topologyInstanceId); + } catch(FailedFetchException e) { + LOG.warn("Failed to fetch from partition " + partition); + if(lastMeta==null) { + return null; + } else { + Map ret = new HashMap(); + ret.put("offset", lastMeta.get("nextOffset")); + ret.put("nextOffset", lastMeta.get("nextOffset")); + return ret; + } + } + } + + @Override + public long numPartitions() { + StaticHosts hosts = (StaticHosts) _config.hosts; + return hosts.hosts.size() * hosts.partitionsPerHost; + } + + @Override + public void close() { + _connections.close(); + } + } +} diff --git a/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java b/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java new file mode 100644 index 0000000000..10b6f800d1 --- /dev/null +++ b/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java @@ -0,0 +1,112 @@ +package storm.kafka.trident; + +import backtype.storm.task.TopologyContext; +import backtype.storm.tuple.Fields; +import java.util.Map; +import java.util.UUID; +import kafka.api.FetchRequest; +import kafka.javaapi.consumer.SimpleConsumer; +import kafka.javaapi.message.ByteBufferMessageSet; +import kafka.message.MessageAndOffset; +import storm.kafka.KafkaConfig.StaticHosts; +import storm.kafka.StaticPartitionConnections; +import storm.trident.operation.TridentCollector; +import storm.trident.spout.IPartitionedTridentSpout; +import storm.trident.topology.TransactionAttempt; + + +public class TransactionalTridentKafkaSpout implements IPartitionedTridentSpout { + + TridentKafkaConfig _config; + String _topologyInstanceId = UUID.randomUUID().toString(); + + + public TransactionalTridentKafkaSpout(TridentKafkaConfig config) { + _config = config; + } + + class Coordinator implements IPartitionedTridentSpout.Coordinator { + @Override + public long numPartitions() { + return computeNumPartitions(); + } + + @Override + public void close() { + _config.coordinator.close(); + } + + @Override + public boolean isReady(long txid) { + return _config.coordinator.isReady(txid); + } + } + + class Emitter implements IPartitionedTridentSpout.Emitter { + StaticPartitionConnections _connections; + int partitionsPerHost; + + public Emitter() { + _connections = new StaticPartitionConnections(_config); + StaticHosts hosts = (StaticHosts) _config.hosts; + partitionsPerHost = hosts.partitionsPerHost; + } + + @Override + public Map emitPartitionBatchNew(TransactionAttempt attempt, TridentCollector collector, int partition, Map lastMeta) { + SimpleConsumer consumer = _connections.getConsumer(partition); + + return KafkaUtils.emitPartitionBatchNew(_config, partition, consumer, attempt, collector, lastMeta, _topologyInstanceId); + } + + @Override + public void emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, int partition, Map meta) { + String instanceId = (String) meta.get("instanceId"); + if(!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) { + SimpleConsumer consumer = _connections.getConsumer(partition); + long offset = (Long) meta.get("offset"); + long nextOffset = (Long) meta.get("nextOffset"); + ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition % partitionsPerHost, offset, _config.fetchSizeBytes)); + for(MessageAndOffset msg: msgs) { + if(offset == nextOffset) break; + if(offset > nextOffset) { + throw new RuntimeException("Error when re-emitting batch. overshot the end offset"); + } + KafkaUtils.emit(_config, attempt, collector, msg.message()); + offset = msg.offset(); + } + } + } + + @Override + public void close() { + _connections.close(); + } + } + + + @Override + public IPartitionedTridentSpout.Coordinator getCoordinator(Map conf, TopologyContext context) { + return new Coordinator(); + } + + @Override + public IPartitionedTridentSpout.Emitter getEmitter(Map conf, TopologyContext context) { + return new Emitter(); + } + + @Override + public Fields getOutputFields() { + return _config.scheme.getOutputFields(); + } + + private int computeNumPartitions() { + StaticHosts hosts = (StaticHosts) _config.hosts; + return hosts.hosts.size() * hosts.partitionsPerHost; + } + + @Override + public Map getComponentConfiguration() { + return null; + } +} \ No newline at end of file diff --git a/src/jvm/storm/kafka/trident/TridentKafkaConfig.java b/src/jvm/storm/kafka/trident/TridentKafkaConfig.java new file mode 100644 index 0000000000..862a1466be --- /dev/null +++ b/src/jvm/storm/kafka/trident/TridentKafkaConfig.java @@ -0,0 +1,12 @@ +package storm.kafka.trident; + +import storm.kafka.KafkaConfig; + + +public class TridentKafkaConfig extends KafkaConfig { + public TridentKafkaConfig(BrokerHosts hosts, String topic) { + super(hosts, topic); + } + + public IBatchCoordinator coordinator = new DefaultCoordinator(); +} From fbca0982d9d88c2a229bb254ffc38e1c2954df11 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 13 Oct 2012 15:08:18 -0700 Subject: [PATCH 138/349] implemented trident kafka spouts with dynamic brokers --- project.clj | 2 +- .../kafka/DynamicPartitionConnections.java | 16 +++-- src/jvm/storm/kafka/GlobalPartitionId.java | 13 +++- .../storm/kafka/trident/IBrokerReader.java | 13 ++++ src/jvm/storm/kafka/trident/KafkaUtils.java | 38 ++++++++--- .../trident/OpaqueTridentKafkaSpout.java | 55 +++++++++------ .../kafka/trident/StaticBrokerReader.java | 32 +++++++++ .../TransactionalTridentKafkaSpout.java | 67 +++++++++++-------- .../storm/kafka/trident/ZkBrokerReader.java | 37 ++++++++++ 9 files changed, 209 insertions(+), 64 deletions(-) create mode 100644 src/jvm/storm/kafka/trident/IBrokerReader.java create mode 100644 src/jvm/storm/kafka/trident/StaticBrokerReader.java create mode 100644 src/jvm/storm/kafka/trident/ZkBrokerReader.java diff --git a/project.clj b/project.clj index cc8ade6ed3..93a3a50e49 100644 --- a/project.clj +++ b/project.clj @@ -7,7 +7,7 @@ :dependencies [[storm/kafka "0.7.0-incubating" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.8.0"] + :dev-dependencies [[storm "0.8.2-wip12-dev"] [org.clojure/clojure "1.4.0"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) diff --git a/src/jvm/storm/kafka/DynamicPartitionConnections.java b/src/jvm/storm/kafka/DynamicPartitionConnections.java index adf7b5aed1..12a121b0dc 100644 --- a/src/jvm/storm/kafka/DynamicPartitionConnections.java +++ b/src/jvm/storm/kafka/DynamicPartitionConnections.java @@ -18,12 +18,16 @@ public ConnectionInfo(SimpleConsumer consumer) { } Map _connections = new HashMap(); - SpoutConfig _config; + KafkaConfig _config; - public DynamicPartitionConnections(SpoutConfig config) { + public DynamicPartitionConnections(KafkaConfig config) { _config = config; } + public SimpleConsumer register(GlobalPartitionId id) { + return register(id.host, id.partition); + } + public SimpleConsumer register(HostPort host, int partition) { if(!_connections.containsKey(host)) { _connections.put(host, new ConnectionInfo(new SimpleConsumer(host.host, host.port, _config.socketTimeoutMs, _config.bufferSizeBytes))); @@ -36,13 +40,17 @@ public SimpleConsumer register(HostPort host, int partition) { public void unregister(HostPort port, int partition) { ConnectionInfo info = _connections.get(port); info.partitions.remove(partition); - if(info.partitions.size()==0) { + if(info.partitions.isEmpty()) { info.consumer.close(); } _connections.remove(port); } - public void close() { + public void unregister(GlobalPartitionId id) { + return unregister(id.host, id.partition); + } + + public void clear() { for(ConnectionInfo info: _connections.values()) { info.consumer.close(); } diff --git a/src/jvm/storm/kafka/GlobalPartitionId.java b/src/jvm/storm/kafka/GlobalPartitionId.java index 460fc9a8b5..5a63f9529d 100644 --- a/src/jvm/storm/kafka/GlobalPartitionId.java +++ b/src/jvm/storm/kafka/GlobalPartitionId.java @@ -1,9 +1,11 @@ package storm.kafka; +import storm.trident.spout.ISpoutPartition; -public class GlobalPartitionId { - HostPort host; - int partition; + +public class GlobalPartitionId implements ISpoutPartition { + public HostPort host; + public int partition; public GlobalPartitionId(HostPort host, int partition) { this.host = host; @@ -23,6 +25,11 @@ public int hashCode() { @Override public String toString() { + return getId(); + } + + @Override + public String getId() { return host.toString() + ":" + partition; } } diff --git a/src/jvm/storm/kafka/trident/IBrokerReader.java b/src/jvm/storm/kafka/trident/IBrokerReader.java new file mode 100644 index 0000000000..7c76f465f2 --- /dev/null +++ b/src/jvm/storm/kafka/trident/IBrokerReader.java @@ -0,0 +1,13 @@ +package storm.kafka.trident; + +import java.util.List; +import java.util.Map; + + +public interface IBrokerReader { + /** + * Map of host to [port, numPartitions] + */ + Map getCurrentBrokers(); + void close(); +} diff --git a/src/jvm/storm/kafka/trident/KafkaUtils.java b/src/jvm/storm/kafka/trident/KafkaUtils.java index b91f4ccffa..97a9ff427f 100644 --- a/src/jvm/storm/kafka/trident/KafkaUtils.java +++ b/src/jvm/storm/kafka/trident/KafkaUtils.java @@ -2,6 +2,7 @@ import backtype.storm.utils.Utils; import java.net.ConnectException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -10,30 +11,51 @@ import kafka.javaapi.message.ByteBufferMessageSet; import kafka.message.Message; import kafka.message.MessageAndOffset; +import storm.kafka.GlobalPartitionId; +import storm.kafka.HostPort; import storm.kafka.KafkaConfig.StaticHosts; +import storm.kafka.KafkaConfig.ZkHosts; import storm.trident.operation.TridentCollector; -import storm.trident.topology.TransactionAttempt; public class KafkaUtils { + public static IBrokerReader makeBrokerReader(Map stormConf, TridentKafkaConfig conf) { + if(conf.hosts instanceof StaticHosts) { + return new StaticBrokerReader((StaticHosts) conf.hosts); + } else { + return new ZkBrokerReader(stormConf, conf.topic, (ZkHosts) conf.hosts); + } + } + public static List getOrderedPartitions(Map partitions) { + List ret = new ArrayList(); + for(String host: partitions.keySet()) { + List info = partitions.get(host); + long port = (Long) info.get(0); + long numPartitions = (Long) info.get(1); + HostPort hp = new HostPort(host, (int) port); + for(int i=0; i values = config.scheme.deserialize(Utils.toByteArray(msg.payload())); if(values!=null) { collector.emit(values); diff --git a/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java b/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java index d31fdcf759..62f2a34528 100644 --- a/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java +++ b/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java @@ -3,18 +3,19 @@ import backtype.storm.task.TopologyContext; import backtype.storm.tuple.Fields; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.UUID; import kafka.javaapi.consumer.SimpleConsumer; import org.apache.log4j.Logger; -import storm.kafka.KafkaConfig.StaticHosts; -import storm.kafka.StaticPartitionConnections; +import storm.kafka.DynamicPartitionConnections; +import storm.kafka.GlobalPartitionId; import storm.trident.operation.TridentCollector; import storm.trident.spout.IOpaquePartitionedTridentSpout; import storm.trident.topology.TransactionAttempt; -public class OpaqueTridentKafkaSpout implements IOpaquePartitionedTridentSpout { +public class OpaqueTridentKafkaSpout implements IOpaquePartitionedTridentSpout, GlobalPartitionId, Map> { public static final Logger LOG = Logger.getLogger(OpaqueTridentKafkaSpout.class); TridentKafkaConfig _config; @@ -25,13 +26,13 @@ public OpaqueTridentKafkaSpout(TridentKafkaConfig config) { } @Override - public IOpaquePartitionedTridentSpout.Emitter getEmitter(Map conf, TopologyContext context) { + public IOpaquePartitionedTridentSpout.Emitter, GlobalPartitionId, Map> getEmitter(Map conf, TopologyContext context) { return new Emitter(); } @Override - public IOpaquePartitionedTridentSpout.Coordinator getCoordinator(Map map, TopologyContext tc) { - return new Coordinator(); + public IOpaquePartitionedTridentSpout.Coordinator getCoordinator(Map conf, TopologyContext tc) { + return new Coordinator(conf); } @Override @@ -44,7 +45,13 @@ public Map getComponentConfiguration() { return null; } - class Coordinator implements IOpaquePartitionedTridentSpout.Coordinator { + class Coordinator implements IOpaquePartitionedTridentSpout.Coordinator { + IBrokerReader reader; + + public Coordinator(Map conf) { + reader = KafkaUtils.makeBrokerReader(conf, _config); + } + @Override public void close() { _config.coordinator.close(); @@ -54,20 +61,26 @@ public void close() { public boolean isReady(long txid) { return _config.coordinator.isReady(txid); } + + @Override + public Map getPartitionsForBatch() { + return reader.getCurrentBrokers(); + } } - class Emitter implements IOpaquePartitionedTridentSpout.Emitter { - StaticPartitionConnections _connections; + class Emitter implements IOpaquePartitionedTridentSpout.Emitter, GlobalPartitionId, Map> { + DynamicPartitionConnections _connections; + public Emitter() { - _connections = new StaticPartitionConnections(_config); + _connections = new DynamicPartitionConnections(_config); } @Override - public Map emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, int partition, Map lastMeta) { + public Map emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map lastMeta) { try { - SimpleConsumer consumer = _connections.getConsumer(partition); - return KafkaUtils.emitPartitionBatchNew(_config, partition, consumer, attempt, collector, lastMeta, _topologyInstanceId); + SimpleConsumer consumer = _connections.register(partition); + return KafkaUtils.emitPartitionBatchNew(_config, consumer, partition.partition, collector, lastMeta, _topologyInstanceId); } catch(FailedFetchException e) { LOG.warn("Failed to fetch from partition " + partition); if(lastMeta==null) { @@ -82,14 +95,18 @@ public Map emitPartitionBatch(TransactionAttempt attempt, TridentCollector colle } @Override - public long numPartitions() { - StaticHosts hosts = (StaticHosts) _config.hosts; - return hosts.hosts.size() * hosts.partitionsPerHost; + public void close() { + _connections.clear(); } @Override - public void close() { - _connections.close(); - } + public List getOrderedPartitions(Map partitions) { + return KafkaUtils.getOrderedPartitions(partitions); + } + + @Override + public void refreshPartitions(List list) { + _connections.clear(); + } } } diff --git a/src/jvm/storm/kafka/trident/StaticBrokerReader.java b/src/jvm/storm/kafka/trident/StaticBrokerReader.java new file mode 100644 index 0000000000..da552e42cb --- /dev/null +++ b/src/jvm/storm/kafka/trident/StaticBrokerReader.java @@ -0,0 +1,32 @@ +package storm.kafka.trident; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import storm.kafka.HostPort; +import storm.kafka.KafkaConfig.StaticHosts; + + +public class StaticBrokerReader implements IBrokerReader { + + Map brokers = new HashMap(); + + public StaticBrokerReader(StaticHosts hosts) { + for(HostPort hp: hosts.hosts) { + List info = new ArrayList(); + info.add((long) hp.port); + info.add((long) hosts.partitionsPerHost); + brokers.put(hp.host, info); + } + } + + @Override + public Map getCurrentBrokers() { + return brokers; + } + + @Override + public void close() { + } +} diff --git a/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java b/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java index 10b6f800d1..229e70868f 100644 --- a/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java +++ b/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java @@ -2,20 +2,21 @@ import backtype.storm.task.TopologyContext; import backtype.storm.tuple.Fields; +import java.util.List; import java.util.Map; import java.util.UUID; import kafka.api.FetchRequest; import kafka.javaapi.consumer.SimpleConsumer; import kafka.javaapi.message.ByteBufferMessageSet; import kafka.message.MessageAndOffset; -import storm.kafka.KafkaConfig.StaticHosts; -import storm.kafka.StaticPartitionConnections; +import storm.kafka.DynamicPartitionConnections; +import storm.kafka.GlobalPartitionId; import storm.trident.operation.TridentCollector; import storm.trident.spout.IPartitionedTridentSpout; import storm.trident.topology.TransactionAttempt; -public class TransactionalTridentKafkaSpout implements IPartitionedTridentSpout { +public class TransactionalTridentKafkaSpout implements IPartitionedTridentSpout, GlobalPartitionId, Map> { TridentKafkaConfig _config; String _topologyInstanceId = UUID.randomUUID().toString(); @@ -25,12 +26,13 @@ public TransactionalTridentKafkaSpout(TridentKafkaConfig config) { _config = config; } - class Coordinator implements IPartitionedTridentSpout.Coordinator { - @Override - public long numPartitions() { - return computeNumPartitions(); + class Coordinator implements IPartitionedTridentSpout.Coordinator { + IBrokerReader reader; + + public Coordinator(Map conf) { + reader = KafkaUtils.makeBrokerReader(conf, _config); } - + @Override public void close() { _config.coordinator.close(); @@ -40,39 +42,41 @@ public void close() { public boolean isReady(long txid) { return _config.coordinator.isReady(txid); } + + @Override + public Map getPartitionsForBatch() { + return reader.getCurrentBrokers(); + } } - class Emitter implements IPartitionedTridentSpout.Emitter { - StaticPartitionConnections _connections; - int partitionsPerHost; + class Emitter implements IPartitionedTridentSpout.Emitter, GlobalPartitionId, Map> { + DynamicPartitionConnections _connections; public Emitter() { - _connections = new StaticPartitionConnections(_config); - StaticHosts hosts = (StaticHosts) _config.hosts; - partitionsPerHost = hosts.partitionsPerHost; + _connections = new DynamicPartitionConnections(_config); } @Override - public Map emitPartitionBatchNew(TransactionAttempt attempt, TridentCollector collector, int partition, Map lastMeta) { - SimpleConsumer consumer = _connections.getConsumer(partition); + public Map emitPartitionBatchNew(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map lastMeta) { + SimpleConsumer consumer = _connections.register(partition); - return KafkaUtils.emitPartitionBatchNew(_config, partition, consumer, attempt, collector, lastMeta, _topologyInstanceId); + return KafkaUtils.emitPartitionBatchNew(_config, consumer, partition.partition, collector, lastMeta, _topologyInstanceId); } @Override - public void emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, int partition, Map meta) { + public void emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map meta) { String instanceId = (String) meta.get("instanceId"); if(!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) { - SimpleConsumer consumer = _connections.getConsumer(partition); + SimpleConsumer consumer = _connections.register(partition); long offset = (Long) meta.get("offset"); long nextOffset = (Long) meta.get("nextOffset"); - ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition % partitionsPerHost, offset, _config.fetchSizeBytes)); + ByteBufferMessageSet msgs = consumer.fetch(new FetchRequest(_config.topic, partition.partition, offset, _config.fetchSizeBytes)); for(MessageAndOffset msg: msgs) { if(offset == nextOffset) break; if(offset > nextOffset) { throw new RuntimeException("Error when re-emitting batch. overshot the end offset"); } - KafkaUtils.emit(_config, attempt, collector, msg.message()); + KafkaUtils.emit(_config, collector, msg.message()); offset = msg.offset(); } } @@ -80,14 +84,24 @@ public void emitPartitionBatch(TransactionAttempt attempt, TridentCollector coll @Override public void close() { - _connections.close(); + _connections.clear(); + } + + @Override + public List getOrderedPartitions(Map partitions) { + return KafkaUtils.getOrderedPartitions(partitions); + } + + @Override + public void refreshPartitions(List list) { + _connections.clear(); } } @Override public IPartitionedTridentSpout.Coordinator getCoordinator(Map conf, TopologyContext context) { - return new Coordinator(); + return new Coordinator(conf); } @Override @@ -99,12 +113,7 @@ public IPartitionedTridentSpout.Emitter getEmitter(Map conf, TopologyContext con public Fields getOutputFields() { return _config.scheme.getOutputFields(); } - - private int computeNumPartitions() { - StaticHosts hosts = (StaticHosts) _config.hosts; - return hosts.hosts.size() * hosts.partitionsPerHost; - } - + @Override public Map getComponentConfiguration() { return null; diff --git a/src/jvm/storm/kafka/trident/ZkBrokerReader.java b/src/jvm/storm/kafka/trident/ZkBrokerReader.java new file mode 100644 index 0000000000..9f0496d9c6 --- /dev/null +++ b/src/jvm/storm/kafka/trident/ZkBrokerReader.java @@ -0,0 +1,37 @@ +package storm.kafka.trident; + +import java.util.List; +import java.util.Map; +import storm.kafka.DynamicBrokersReader; +import storm.kafka.KafkaConfig.ZkHosts; + + +public class ZkBrokerReader implements IBrokerReader { + + Map cachedBrokers; + DynamicBrokersReader reader; + long lastRefreshTimeMs; + long refreshMillis; + + public ZkBrokerReader(Map conf, String topic, ZkHosts hosts) { + reader = new DynamicBrokersReader(conf, hosts.brokerZkStr, hosts.brokerZkPath, topic); + cachedBrokers = reader.getBrokerInfo(); + lastRefreshTimeMs = System.currentTimeMillis(); + refreshMillis = hosts.refreshFreqSecs * 1000L; + } + + @Override + public Map getCurrentBrokers() { + long currTime = System.currentTimeMillis(); + if(currTime > lastRefreshTimeMs + refreshMillis) { + cachedBrokers = reader.getBrokerInfo(); + lastRefreshTimeMs = currTime; + } + return cachedBrokers; + } + + @Override + public void close() { + reader.close(); + } +} From 7e074cfe5c88dd927c2f87bff985e93228ff0b13 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 13 Oct 2012 15:08:56 -0700 Subject: [PATCH 139/349] bump version --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 93a3a50e49..42660d69bd 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.8.0-wip2" +(defproject storm/storm-kafka "0.9.0-wip1" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} From 3641921f21d20a631cf1d24c5642d985b166accf Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 13 Oct 2012 15:10:29 -0700 Subject: [PATCH 140/349] depend on 0.9.0-wip1 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 42660d69bd..3e06308974 100644 --- a/project.clj +++ b/project.clj @@ -7,7 +7,7 @@ :dependencies [[storm/kafka "0.7.0-incubating" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.8.2-wip12-dev"] + :dev-dependencies [[storm "0.9.0-wip1"] [org.clojure/clojure "1.4.0"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) From 7fd0575aa305d00a88065a39d470dcfab28e245d Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Sat, 13 Oct 2012 15:16:15 -0700 Subject: [PATCH 141/349] fix typo --- src/jvm/storm/kafka/DynamicPartitionConnections.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jvm/storm/kafka/DynamicPartitionConnections.java b/src/jvm/storm/kafka/DynamicPartitionConnections.java index 12a121b0dc..9e48cf4d76 100644 --- a/src/jvm/storm/kafka/DynamicPartitionConnections.java +++ b/src/jvm/storm/kafka/DynamicPartitionConnections.java @@ -47,7 +47,7 @@ public void unregister(HostPort port, int partition) { } public void unregister(GlobalPartitionId id) { - return unregister(id.host, id.partition); + unregister(id.host, id.partition); } public void clear() { From ae825eb9713913cf2caef742fa4dd3a52ecceec3 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 25 Oct 2012 11:41:42 -0700 Subject: [PATCH 142/349] update kafkaspout to write metadata into zookeeper so that lag monitoring can be done --- src/jvm/storm/kafka/trident/KafkaUtils.java | 20 ++++++++++++++----- .../trident/OpaqueTridentKafkaSpout.java | 16 +++++++++++---- .../TransactionalTridentKafkaSpout.java | 9 ++++++--- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/jvm/storm/kafka/trident/KafkaUtils.java b/src/jvm/storm/kafka/trident/KafkaUtils.java index 97a9ff427f..24beae9247 100644 --- a/src/jvm/storm/kafka/trident/KafkaUtils.java +++ b/src/jvm/storm/kafka/trident/KafkaUtils.java @@ -1,6 +1,7 @@ package storm.kafka.trident; import backtype.storm.utils.Utils; +import com.google.common.collect.ImmutableMap; import java.net.ConnectException; import java.util.ArrayList; import java.util.HashMap; @@ -40,22 +41,27 @@ public static List getOrderedPartitions(Map par return ret; } - public static Map emitPartitionBatchNew(TridentKafkaConfig config, SimpleConsumer consumer, int partition, TridentCollector collector, Map lastMeta, String topologyInstanceId) { + public static Map emitPartitionBatchNew(TridentKafkaConfig config, SimpleConsumer consumer, GlobalPartitionId partition, TridentCollector collector, Map lastMeta, String topologyInstanceId, String topologyName) { long offset; if(lastMeta!=null) { - if(config.forceFromStart && !topologyInstanceId.equals(lastMeta.get("instanceId"))) { - offset = consumer.getOffsetsBefore(config.topic, partition, config.startOffsetTime, 1)[0]; + String lastInstanceId = null; + Map lastTopoMeta = (Map) lastMeta.get("topology"); + if(lastTopoMeta!=null) { + lastInstanceId = (String) lastTopoMeta.get("id"); + } + if(config.forceFromStart && !topologyInstanceId.equals(lastInstanceId)) { + offset = consumer.getOffsetsBefore(config.topic, partition.partition, config.startOffsetTime, 1)[0]; } else { offset = (Long) lastMeta.get("nextOffset"); } } else { long startTime = -1; if(config.forceFromStart) startTime = config.startOffsetTime; - offset = consumer.getOffsetsBefore(config.topic, partition, startTime, 1)[0]; + offset = consumer.getOffsetsBefore(config.topic, partition.partition, startTime, 1)[0]; } ByteBufferMessageSet msgs; try { - msgs = consumer.fetch(new FetchRequest(config.topic, partition, offset, config.fetchSizeBytes)); + msgs = consumer.fetch(new FetchRequest(config.topic, partition.partition, offset, config.fetchSizeBytes)); } catch(Exception e) { if(e instanceof ConnectException) { throw new FailedFetchException(e); @@ -72,6 +78,10 @@ public static Map emitPartitionBatchNew(TridentKafkaConfig config, SimpleConsume newMeta.put("offset", offset); newMeta.put("nextOffset", endoffset); newMeta.put("instanceId", topologyInstanceId); + newMeta.put("partition", partition.partition); + newMeta.put("broker", ImmutableMap.of("host", partition.host.host, "port", partition.host.port)); + newMeta.put("topic", config.topic); + newMeta.put("topology", ImmutableMap.of("name", topologyName, "id", topologyInstanceId)); return newMeta; } diff --git a/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java b/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java index 62f2a34528..5695636924 100644 --- a/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java +++ b/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java @@ -1,7 +1,9 @@ package storm.kafka.trident; +import backtype.storm.Config; import backtype.storm.task.TopologyContext; import backtype.storm.tuple.Fields; +import com.google.common.collect.ImmutableMap; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,7 +29,7 @@ public OpaqueTridentKafkaSpout(TridentKafkaConfig config) { @Override public IOpaquePartitionedTridentSpout.Emitter, GlobalPartitionId, Map> getEmitter(Map conf, TopologyContext context) { - return new Emitter(); + return new Emitter(conf); } @Override @@ -70,17 +72,18 @@ public Map getPartitionsForBatch() { class Emitter implements IOpaquePartitionedTridentSpout.Emitter, GlobalPartitionId, Map> { DynamicPartitionConnections _connections; + String _topologyName; - - public Emitter() { + public Emitter(Map conf) { _connections = new DynamicPartitionConnections(_config); + _topologyName = (String) conf.get(Config.TOPOLOGY_NAME); } @Override public Map emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map lastMeta) { try { SimpleConsumer consumer = _connections.register(partition); - return KafkaUtils.emitPartitionBatchNew(_config, consumer, partition.partition, collector, lastMeta, _topologyInstanceId); + return KafkaUtils.emitPartitionBatchNew(_config, consumer, partition, collector, lastMeta, _topologyInstanceId, _topologyName); } catch(FailedFetchException e) { LOG.warn("Failed to fetch from partition " + partition); if(lastMeta==null) { @@ -89,6 +92,11 @@ public Map emitPartitionBatch(TransactionAttempt attempt, TridentCollector colle Map ret = new HashMap(); ret.put("offset", lastMeta.get("nextOffset")); ret.put("nextOffset", lastMeta.get("nextOffset")); + ret.put("partition", partition.partition); + ret.put("broker", ImmutableMap.of("host", partition.host.host, "port", partition.host.port)); + ret.put("topic", _config.topic); + ret.put("topology", ImmutableMap.of("name", _topologyName, "id", _topologyInstanceId)); + return ret; } } diff --git a/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java b/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java index 229e70868f..cd9dbd087c 100644 --- a/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java +++ b/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java @@ -1,5 +1,6 @@ package storm.kafka.trident; +import backtype.storm.Config; import backtype.storm.task.TopologyContext; import backtype.storm.tuple.Fields; import java.util.List; @@ -51,16 +52,18 @@ public Map getPartitionsForBatch() { class Emitter implements IPartitionedTridentSpout.Emitter, GlobalPartitionId, Map> { DynamicPartitionConnections _connections; + String _topologyName; - public Emitter() { + public Emitter(Map conf) { _connections = new DynamicPartitionConnections(_config); + _topologyName = (String) conf.get(Config.TOPOLOGY_NAME); } @Override public Map emitPartitionBatchNew(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map lastMeta) { SimpleConsumer consumer = _connections.register(partition); - return KafkaUtils.emitPartitionBatchNew(_config, consumer, partition.partition, collector, lastMeta, _topologyInstanceId); + return KafkaUtils.emitPartitionBatchNew(_config, consumer, partition, collector, lastMeta, _topologyInstanceId, _topologyName); } @Override @@ -106,7 +109,7 @@ public IPartitionedTridentSpout.Coordinator getCoordinator(Map conf, TopologyCon @Override public IPartitionedTridentSpout.Emitter getEmitter(Map conf, TopologyContext context) { - return new Emitter(); + return new Emitter(conf); } @Override From fd0d33161bd3c72e8dc650719d4b6b9cdcf3fa78 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Thu, 25 Oct 2012 14:26:51 -0700 Subject: [PATCH 143/349] Update README.markdown --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 6ea5cc8a31..3191b2ee64 100644 --- a/README.markdown +++ b/README.markdown @@ -10,7 +10,7 @@ More information about Storm can be found on the [project page](http://github.co ## Running an example with Leiningen -Install Leiningen by following the installation instructions [here](https://github.com/technomancy/leiningen). +Install Leiningen by following the installation instructions [here](https://github.com/technomancy/leiningen). The storm-starter build uses Leiningen 1.7.1. ### To run a Java example: From 63aaad01981cd1683a74ce81189ebf4a7a420fb0 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 2 Nov 2012 15:33:30 -0700 Subject: [PATCH 144/349] upgrade to 0.9.0's new logging system --- project.clj | 4 +++- src/jvm/storm/kafka/KafkaSpout.java | 6 +++--- src/jvm/storm/kafka/PartitionManager.java | 5 +++-- src/jvm/storm/kafka/ZkCoordinator.java | 5 +++-- src/jvm/storm/kafka/ZkState.java | 5 +++-- src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java | 5 +++-- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/project.clj b/project.clj index 3e06308974..67443d447b 100644 --- a/project.clj +++ b/project.clj @@ -7,7 +7,9 @@ :dependencies [[storm/kafka "0.7.0-incubating" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.9.0-wip1"] + :dev-dependencies [[storm "0.9.0-wip3"] + [org.slf4j/log4j-over-slf4j "1.6.6"] + [ch.qos.logback/logback-classic "1.0.6"] [org.clojure/clojure "1.4.0"]] :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] ) diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index 6027cce656..525a1b3878 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -5,13 +5,13 @@ import backtype.storm.task.TopologyContext; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseRichSpout; -import java.io.Serializable; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; -import org.apache.log4j.Logger; import kafka.message.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import storm.kafka.PartitionManager.KafkaMessageId; // TODO: need to add blacklisting @@ -33,7 +33,7 @@ static enum EmitState { NO_EMITTED } - public static final Logger LOG = Logger.getLogger(KafkaSpout.class); + public static final Logger LOG = LoggerFactory.getLogger(KafkaSpout.class); String _uuid = UUID.randomUUID().toString(); SpoutConfig _spoutConfig; diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 8464c5362a..19e8ad99b0 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -9,12 +9,13 @@ import kafka.javaapi.consumer.SimpleConsumer; import kafka.javaapi.message.ByteBufferMessageSet; import kafka.message.MessageAndOffset; -import org.apache.log4j.Logger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import storm.kafka.KafkaSpout.EmitState; import storm.kafka.KafkaSpout.MessageAndRealOffset; public class PartitionManager { - public static final Logger LOG = Logger.getLogger(PartitionManager.class); + public static final Logger LOG = LoggerFactory.getLogger(PartitionManager.class); static class KafkaMessageId { public GlobalPartitionId partition; diff --git a/src/jvm/storm/kafka/ZkCoordinator.java b/src/jvm/storm/kafka/ZkCoordinator.java index 4b4eaff3f5..4649a371f3 100644 --- a/src/jvm/storm/kafka/ZkCoordinator.java +++ b/src/jvm/storm/kafka/ZkCoordinator.java @@ -6,11 +6,12 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.log4j.Logger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import storm.kafka.KafkaConfig.ZkHosts; public class ZkCoordinator implements PartitionCoordinator { - public static Logger LOG = Logger.getLogger(ZkCoordinator.class); + public static Logger LOG = LoggerFactory.getLogger(ZkCoordinator.class); SpoutConfig _spoutConfig; int _taskIndex; diff --git a/src/jvm/storm/kafka/ZkState.java b/src/jvm/storm/kafka/ZkState.java index a72c5d6e6e..567728007f 100644 --- a/src/jvm/storm/kafka/ZkState.java +++ b/src/jvm/storm/kafka/ZkState.java @@ -9,12 +9,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.log4j.Logger; import org.apache.zookeeper.CreateMode; import org.json.simple.JSONValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ZkState { - public static final Logger LOG = Logger.getLogger(ZkState.class); + public static final Logger LOG = LoggerFactory.getLogger(ZkState.class); CuratorFramework _curator; private CuratorFramework newCurator(Map stateConf) throws Exception { diff --git a/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java b/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java index 5695636924..a7a9c3883e 100644 --- a/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java +++ b/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java @@ -9,7 +9,8 @@ import java.util.Map; import java.util.UUID; import kafka.javaapi.consumer.SimpleConsumer; -import org.apache.log4j.Logger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import storm.kafka.DynamicPartitionConnections; import storm.kafka.GlobalPartitionId; import storm.trident.operation.TridentCollector; @@ -18,7 +19,7 @@ public class OpaqueTridentKafkaSpout implements IOpaquePartitionedTridentSpout, GlobalPartitionId, Map> { - public static final Logger LOG = Logger.getLogger(OpaqueTridentKafkaSpout.class); + public static final Logger LOG = LoggerFactory.getLogger(OpaqueTridentKafkaSpout.class); TridentKafkaConfig _config; String _topologyInstanceId = UUID.randomUUID().toString(); From f5938c06a55c523b39ce12aabec3c6f422fe8b3d Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 2 Nov 2012 15:36:11 -0700 Subject: [PATCH 145/349] 0.9.0-wip2 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 67443d447b..651a9f5670 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.9.0-wip1" +(defproject storm/storm-kafka "0.9.0-wip2" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} From 6cc99e37d1c6c4f51394f8bf7119d1ee801f159c Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Tue, 13 Nov 2012 16:50:40 -0800 Subject: [PATCH 146/349] make sure kafka partitions are always ordered the same way --- src/jvm/storm/kafka/trident/KafkaUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/jvm/storm/kafka/trident/KafkaUtils.java b/src/jvm/storm/kafka/trident/KafkaUtils.java index 24beae9247..59f2b1bfdf 100644 --- a/src/jvm/storm/kafka/trident/KafkaUtils.java +++ b/src/jvm/storm/kafka/trident/KafkaUtils.java @@ -7,6 +7,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import kafka.api.FetchRequest; import kafka.javaapi.consumer.SimpleConsumer; import kafka.javaapi.message.ByteBufferMessageSet; @@ -29,7 +30,7 @@ public static IBrokerReader makeBrokerReader(Map stormConf, TridentKafkaConfig c public static List getOrderedPartitions(Map partitions) { List ret = new ArrayList(); - for(String host: partitions.keySet()) { + for(String host: new TreeMap(partitions).keySet()) { List info = partitions.get(host); long port = (Long) info.get(0); long numPartitions = (Long) info.get(1); From d6e9e63512544d1cf5e34c83af1e9cc7d8dfd4b0 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Tue, 13 Nov 2012 16:52:48 -0800 Subject: [PATCH 147/349] 0.9.0-wip3 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 651a9f5670..6c06f02f5a 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.9.0-wip2" +(defproject storm/storm-kafka "0.9.0-wip3" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} From f2d5b39551d6980c96b87d0c2f0309a8ff104218 Mon Sep 17 00:00:00 2001 From: Sam Ritchie Date: Thu, 29 Nov 2012 17:02:40 -0800 Subject: [PATCH 148/349] multischeme. --- src/jvm/storm/kafka/KafkaConfig.java | 7 +++--- src/jvm/storm/kafka/KafkaSpout.java | 3 ++- src/jvm/storm/kafka/PartitionManager.java | 26 ++++++++++----------- src/jvm/storm/kafka/TestTopology.java | 3 ++- src/jvm/storm/kafka/trident/KafkaUtils.java | 22 ++++++++++------- 5 files changed, 34 insertions(+), 27 deletions(-) diff --git a/src/jvm/storm/kafka/KafkaConfig.java b/src/jvm/storm/kafka/KafkaConfig.java index 943cef1acd..cd40403d8f 100644 --- a/src/jvm/storm/kafka/KafkaConfig.java +++ b/src/jvm/storm/kafka/KafkaConfig.java @@ -1,11 +1,12 @@ package storm.kafka; -import backtype.storm.spout.RawScheme; -import backtype.storm.spout.Scheme; import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import backtype.storm.spout.RawScheme; +import backtype.storm.spout.Scheme; + public class KafkaConfig implements Serializable { public static interface BrokerHosts extends Serializable { @@ -49,7 +50,7 @@ public ZkHosts(String brokerZkStr, String brokerZkPath) { public int fetchSizeBytes = 1024*1024; public int socketTimeoutMs = 10000; public int bufferSizeBytes = 1024*1024; - public Scheme scheme = new RawScheme(); + public MultiScheme scheme = new RawMultiScheme(); public String topic; public long startOffsetTime = -2; public boolean forceFromStart = false; diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index 525a1b3878..1eed964304 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -1,6 +1,7 @@ package storm.kafka; import backtype.storm.Config; +import backtype.storm.spout.SchemeToMultiScheme; import backtype.storm.spout.SpoutOutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.topology.OutputFieldsDeclarer; @@ -143,7 +144,7 @@ public static void main(String[] args) { // List hosts = new ArrayList(); // hosts.add("localhost"); // SpoutConfig spoutConf = SpoutConfig.fromHostStrings(hosts, 8, "clicks", "/kafkastorm", "id"); -// spoutConf.scheme = new StringScheme(); +// spoutConf.scheme = new SchemeToMultiScheme(new StringScheme()); // spoutConf.forceStartOffsetTime(-2); // // // spoutConf.zkServers = new ArrayList() {{ diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 19e8ad99b0..4424b649d7 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -14,7 +14,7 @@ import storm.kafka.KafkaSpout.EmitState; import storm.kafka.KafkaSpout.MessageAndRealOffset; -public class PartitionManager { +public class PartitionManager { public static final Logger LOG = LoggerFactory.getLogger(PartitionManager.class); static class KafkaMessageId { @@ -25,8 +25,8 @@ public KafkaMessageId(GlobalPartitionId partition, long offset) { this.partition = partition; this.offset = offset; } - } - + } + Long _emittedToOffset; SortedSet _pending = new TreeSet(); Long _committedTo; @@ -38,7 +38,7 @@ public KafkaMessageId(GlobalPartitionId partition, long offset) { DynamicPartitionConnections _connections; ZkState _state; Map _stormConf; - + public PartitionManager(DynamicPartitionConnections connections, String topologyInstanceId, ZkState state, Map stormConf, SpoutConfig spoutConfig, GlobalPartitionId id) { _partition = id; _connections = connections; @@ -50,15 +50,15 @@ public PartitionManager(DynamicPartitionConnections connections, String topology String jsonTopologyId = null; Long jsonOffset = null; - try { + try { Map json = _state.readJSON(committedPath()); if(json != null) { jsonTopologyId = (String)((Map)json.get("topology")).get("id"); jsonOffset = (Long)json.get("offset"); } } - catch(Throwable e) { - LOG.warn("Error reading and/or parsing at ZkNode: " + committedPath(), e); + catch(Throwable e) { + LOG.warn("Error reading and/or parsing at ZkNode: " + committedPath(), e); } if(!topologyInstanceId.equals(jsonTopologyId) && spoutConfig.forceFromStart) { @@ -84,9 +84,10 @@ public EmitState next(SpoutOutputCollector collector) { if(toEmit==null) { return EmitState.NO_EMITTED; } - List tup = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.msg.payload())); - if(tup!=null) { - collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset)); + Iterator> tups = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.msg.payload())); + if(tups!=null) { + for(List tup: tups) + collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset)); break; } else { ack(toEmit.offset); @@ -146,7 +147,7 @@ public void commit() { LOG.info("Writing committed offset to ZK: " + committedTo); Map data = (Map)ImmutableMap.builder() - .put("topology", ImmutableMap.of("id", _topologyInstanceId, + .put("topology", ImmutableMap.of("id", _topologyInstanceId, "name", _stormConf.get(Config.TOPOLOGY_NAME))) .put("offset", committedTo) .put("partition", _partition.partition) @@ -164,9 +165,8 @@ public void commit() { private String committedPath() { return _spoutConfig.zkRoot + "/" + _spoutConfig.id + "/" + _partition; } - + public void close() { _connections.unregister(_partition.host, _partition.partition); } } - diff --git a/src/jvm/storm/kafka/TestTopology.java b/src/jvm/storm/kafka/TestTopology.java index 4fa6504d29..4647f504e7 100644 --- a/src/jvm/storm/kafka/TestTopology.java +++ b/src/jvm/storm/kafka/TestTopology.java @@ -2,6 +2,7 @@ import backtype.storm.Config; import backtype.storm.LocalCluster; +import backtype.storm.spout.SchemeToMultiScheme; import backtype.storm.topology.BasicOutputCollector; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseBasicBolt; @@ -28,7 +29,7 @@ public static void main(String [] args) throws Exception { // List hosts = new ArrayList(); // hosts.add("localhost"); // KafkaConfig kafkaConf = new KafkaConfig(StaticHosts.fromHostString(hosts, 3), "test"); -// kafkaConf.scheme = new StringScheme(); +// kafkaConf.scheme = new SchemeToMultiScheme(new StringScheme()); // LocalCluster cluster = new LocalCluster(); // TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("id", "spout", // new OpaqueTransactionalKafkaSpout(kafkaConf), 1); diff --git a/src/jvm/storm/kafka/trident/KafkaUtils.java b/src/jvm/storm/kafka/trident/KafkaUtils.java index 59f2b1bfdf..91aa83dd39 100644 --- a/src/jvm/storm/kafka/trident/KafkaUtils.java +++ b/src/jvm/storm/kafka/trident/KafkaUtils.java @@ -1,13 +1,16 @@ package storm.kafka.trident; -import backtype.storm.utils.Utils; -import com.google.common.collect.ImmutableMap; import java.net.ConnectException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TreeMap; + +import com.google.common.collect.ImmutableMap; + +import backtype.storm.utils.Utils; import kafka.api.FetchRequest; import kafka.javaapi.consumer.SimpleConsumer; import kafka.javaapi.message.ByteBufferMessageSet; @@ -27,7 +30,7 @@ public static IBrokerReader makeBrokerReader(Map stormConf, TridentKafkaConfig c return new ZkBrokerReader(stormConf, conf.topic, (ZkHosts) conf.hosts); } } - + public static List getOrderedPartitions(Map partitions) { List ret = new ArrayList(); for(String host: new TreeMap(partitions).keySet()) { @@ -41,7 +44,7 @@ public static List getOrderedPartitions(Map par } return ret; } - + public static Map emitPartitionBatchNew(TridentKafkaConfig config, SimpleConsumer consumer, GlobalPartitionId partition, TridentCollector collector, Map lastMeta, String topologyInstanceId, String topologyName) { long offset; if(lastMeta!=null) { @@ -53,7 +56,7 @@ public static Map emitPartitionBatchNew(TridentKafkaConfig config, SimpleConsume if(config.forceFromStart && !topologyInstanceId.equals(lastInstanceId)) { offset = consumer.getOffsetsBefore(config.topic, partition.partition, config.startOffsetTime, 1)[0]; } else { - offset = (Long) lastMeta.get("nextOffset"); + offset = (Long) lastMeta.get("nextOffset"); } } else { long startTime = -1; @@ -82,14 +85,15 @@ public static Map emitPartitionBatchNew(TridentKafkaConfig config, SimpleConsume newMeta.put("partition", partition.partition); newMeta.put("broker", ImmutableMap.of("host", partition.host.host, "port", partition.host.port)); newMeta.put("topic", config.topic); - newMeta.put("topology", ImmutableMap.of("name", topologyName, "id", topologyInstanceId)); + newMeta.put("topology", ImmutableMap.of("name", topologyName, "id", topologyInstanceId)); return newMeta; } - + public static void emit(TridentKafkaConfig config, TridentCollector collector, Message msg) { - List values = config.scheme.deserialize(Utils.toByteArray(msg.payload())); + Iterator> values = config.scheme.deserialize(Utils.toByteArray(msg.payload())); if(values!=null) { - collector.emit(values); + for(List value: values) + collector.emit(value); } } } From fecf760a4dbcf4f7f5af13b401345a93bb99417d Mon Sep 17 00:00:00 2001 From: Sam Ritchie Date: Mon, 3 Dec 2012 17:58:09 -0800 Subject: [PATCH 149/349] fix typo. --- src/jvm/storm/kafka/KafkaConfig.java | 4 ++-- src/jvm/storm/kafka/KafkaSpout.java | 17 +++++++++-------- src/jvm/storm/kafka/TestTopology.java | 9 +-------- src/jvm/storm/kafka/trident/KafkaUtils.java | 3 ++- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/jvm/storm/kafka/KafkaConfig.java b/src/jvm/storm/kafka/KafkaConfig.java index cd40403d8f..9e485f8970 100644 --- a/src/jvm/storm/kafka/KafkaConfig.java +++ b/src/jvm/storm/kafka/KafkaConfig.java @@ -4,8 +4,8 @@ import java.util.ArrayList; import java.util.List; -import backtype.storm.spout.RawScheme; -import backtype.storm.spout.Scheme; +import backtype.storm.spout.MultiScheme; +import backtype.storm.spout.RawMultiScheme; public class KafkaConfig implements Serializable { public static interface BrokerHosts extends Serializable { diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index 1eed964304..2e791067e0 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -1,18 +1,19 @@ package storm.kafka; -import backtype.storm.Config; -import backtype.storm.spout.SchemeToMultiScheme; -import backtype.storm.spout.SpoutOutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseRichSpout; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; -import kafka.message.Message; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import backtype.storm.Config; +import backtype.storm.spout.SpoutOutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseRichSpout; +import kafka.message.Message; import storm.kafka.PartitionManager.KafkaMessageId; // TODO: need to add blacklisting @@ -144,7 +145,7 @@ public static void main(String[] args) { // List hosts = new ArrayList(); // hosts.add("localhost"); // SpoutConfig spoutConf = SpoutConfig.fromHostStrings(hosts, 8, "clicks", "/kafkastorm", "id"); -// spoutConf.scheme = new SchemeToMultiScheme(new StringScheme()); +// spoutConf.scheme = new SchemeAsMultiScheme(new StringScheme()); // spoutConf.forceStartOffsetTime(-2); // // // spoutConf.zkServers = new ArrayList() {{ diff --git a/src/jvm/storm/kafka/TestTopology.java b/src/jvm/storm/kafka/TestTopology.java index 4647f504e7..bf4612d2e7 100644 --- a/src/jvm/storm/kafka/TestTopology.java +++ b/src/jvm/storm/kafka/TestTopology.java @@ -1,16 +1,9 @@ package storm.kafka; -import backtype.storm.Config; -import backtype.storm.LocalCluster; -import backtype.storm.spout.SchemeToMultiScheme; import backtype.storm.topology.BasicOutputCollector; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseBasicBolt; -import backtype.storm.transactional.TransactionalTopologyBuilder; import backtype.storm.tuple.Tuple; -import java.util.ArrayList; -import java.util.List; -import storm.kafka.KafkaConfig.StaticHosts; public class TestTopology { public static class PrinterBolt extends BaseBasicBolt { @@ -29,7 +22,7 @@ public static void main(String [] args) throws Exception { // List hosts = new ArrayList(); // hosts.add("localhost"); // KafkaConfig kafkaConf = new KafkaConfig(StaticHosts.fromHostString(hosts, 3), "test"); -// kafkaConf.scheme = new SchemeToMultiScheme(new StringScheme()); +// kafkaConf.scheme = new SchemeAsMultiScheme(new StringScheme()); // LocalCluster cluster = new LocalCluster(); // TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("id", "spout", // new OpaqueTransactionalKafkaSpout(kafkaConf), 1); diff --git a/src/jvm/storm/kafka/trident/KafkaUtils.java b/src/jvm/storm/kafka/trident/KafkaUtils.java index 91aa83dd39..5af1742e9d 100644 --- a/src/jvm/storm/kafka/trident/KafkaUtils.java +++ b/src/jvm/storm/kafka/trident/KafkaUtils.java @@ -90,7 +90,8 @@ public static Map emitPartitionBatchNew(TridentKafkaConfig config, SimpleConsume } public static void emit(TridentKafkaConfig config, TridentCollector collector, Message msg) { - Iterator> values = config.scheme.deserialize(Utils.toByteArray(msg.payload())); + Iterable> values = + config.scheme.deserialize(Utils.toByteArray(msg.payload())); if(values!=null) { for(List value: values) collector.emit(value); From f5bdc720f50a0c46e90f0085c10217f2a6a3249f Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Fri, 7 Dec 2012 12:31:03 +0100 Subject: [PATCH 150/349] Completely refactor Rolling Count example The new implementation separates the various concerns formerly mixed up in the example code. Also, we are now using tick tuples (introduced in Storm 0.8) instead of spawning manual threads for carrying out periodical tasks. Lastly, we add 192 unit tests for the new implementation that brings the test coverage for the Rolling Count from 0% to almost 100%. Note: Adding those unit tests required changes to the build (m2-pom.xml), notably new test dependencies and moving the existing Java code from src/jvm/* to src/jvm/main/*. The latter was required so that the test runner triggered by Maven can tell code (src/jvm/main) and tests (src/jvm/test) apart. --- m2-pom.xml | 31 ++- .../storm/starter/BasicDRPCTopology.java | 0 .../storm/starter/ExclamationTopology.java | 0 .../{ => main}/storm/starter/ManualDRPC.java | 0 .../storm/starter/PrintSampleStream.java | 0 .../storm/starter/ReachTopology.java | 0 .../main/storm/starter/RollingTopWords.java | 61 +++++ .../storm/starter/SingleJoinExample.java | 0 .../starter/TransactionalGlobalCount.java | 0 .../storm/starter/TransactionalWords.java | 0 .../storm/starter/WordCountTopology.java | 0 .../starter/bolt/AbstractRankerBolt.java | 95 ++++++++ .../bolt/IntermediateRankingsBolt.java | 43 ++++ .../storm/starter/bolt/PrinterBolt.java | 0 .../storm/starter/bolt/RollingCountBolt.java | 126 ++++++++++ .../storm/starter/bolt/SingleJoinBolt.java | 0 .../storm/starter/bolt/TotalRankingsBolt.java | 43 ++++ .../starter/spout/RandomSentenceSpout.java | 0 .../starter/spout/TwitterSampleSpout.java | 0 .../tools/NthLastModifiedTimeTracker.java | 55 +++++ .../main/storm/starter/tools/Rankable.java | 9 + .../tools/RankableObjectWithFields.java | 117 +++++++++ .../main/storm/starter/tools/Rankings.java | 93 ++++++++ .../starter/tools/SlidingWindowCounter.java | 102 ++++++++ .../storm/starter/tools/SlotBasedCounter.java | 98 ++++++++ .../storm/starter/trident/TridentReach.java | 0 .../starter/trident/TridentWordCount.java | 0 .../main/storm/starter/util/StormRunner.java | 22 ++ .../main/storm/starter/util/TupleHelpers.java | 16 ++ src/jvm/storm/starter/RollingTopWords.java | 48 ---- src/jvm/storm/starter/bolt/MergeObjects.java | 83 ------- src/jvm/storm/starter/bolt/RankObjects.java | 76 ------ .../starter/bolt/RollingCountObjects.java | 110 --------- .../bolt/IntermediateRankingsBoltTest.java | 135 +++++++++++ .../starter/bolt/RollingCountBoltTest.java | 101 ++++++++ .../starter/bolt/TotalRankingsBoltTest.java | 135 +++++++++++ .../storm/starter/tools/MockTupleHelpers.java | 23 ++ .../tools/NthLastModifiedTimeTrackerTest.java | 112 +++++++++ .../tools/RankableObjectWithFieldsTest.java | 225 ++++++++++++++++++ .../storm/starter/tools/RankingsTest.java | 206 ++++++++++++++++ .../tools/SlidingWindowCounterTest.java | 88 +++++++ .../starter/tools/SlotBasedCounterTest.java | 163 +++++++++++++ 42 files changed, 2098 insertions(+), 318 deletions(-) rename src/jvm/{ => main}/storm/starter/BasicDRPCTopology.java (100%) rename src/jvm/{ => main}/storm/starter/ExclamationTopology.java (100%) rename src/jvm/{ => main}/storm/starter/ManualDRPC.java (100%) rename src/jvm/{ => main}/storm/starter/PrintSampleStream.java (100%) rename src/jvm/{ => main}/storm/starter/ReachTopology.java (100%) create mode 100644 src/jvm/main/storm/starter/RollingTopWords.java rename src/jvm/{ => main}/storm/starter/SingleJoinExample.java (100%) rename src/jvm/{ => main}/storm/starter/TransactionalGlobalCount.java (100%) rename src/jvm/{ => main}/storm/starter/TransactionalWords.java (100%) rename src/jvm/{ => main}/storm/starter/WordCountTopology.java (100%) create mode 100644 src/jvm/main/storm/starter/bolt/AbstractRankerBolt.java create mode 100644 src/jvm/main/storm/starter/bolt/IntermediateRankingsBolt.java rename src/jvm/{ => main}/storm/starter/bolt/PrinterBolt.java (100%) create mode 100644 src/jvm/main/storm/starter/bolt/RollingCountBolt.java rename src/jvm/{ => main}/storm/starter/bolt/SingleJoinBolt.java (100%) create mode 100644 src/jvm/main/storm/starter/bolt/TotalRankingsBolt.java rename src/jvm/{ => main}/storm/starter/spout/RandomSentenceSpout.java (100%) rename src/jvm/{ => main}/storm/starter/spout/TwitterSampleSpout.java (100%) create mode 100644 src/jvm/main/storm/starter/tools/NthLastModifiedTimeTracker.java create mode 100644 src/jvm/main/storm/starter/tools/Rankable.java create mode 100644 src/jvm/main/storm/starter/tools/RankableObjectWithFields.java create mode 100644 src/jvm/main/storm/starter/tools/Rankings.java create mode 100644 src/jvm/main/storm/starter/tools/SlidingWindowCounter.java create mode 100644 src/jvm/main/storm/starter/tools/SlotBasedCounter.java rename src/jvm/{ => main}/storm/starter/trident/TridentReach.java (100%) rename src/jvm/{ => main}/storm/starter/trident/TridentWordCount.java (100%) create mode 100644 src/jvm/main/storm/starter/util/StormRunner.java create mode 100644 src/jvm/main/storm/starter/util/TupleHelpers.java delete mode 100644 src/jvm/storm/starter/RollingTopWords.java delete mode 100644 src/jvm/storm/starter/bolt/MergeObjects.java delete mode 100644 src/jvm/storm/starter/bolt/RankObjects.java delete mode 100644 src/jvm/storm/starter/bolt/RollingCountObjects.java create mode 100644 src/jvm/test/storm/starter/bolt/IntermediateRankingsBoltTest.java create mode 100644 src/jvm/test/storm/starter/bolt/RollingCountBoltTest.java create mode 100644 src/jvm/test/storm/starter/bolt/TotalRankingsBoltTest.java create mode 100644 src/jvm/test/storm/starter/tools/MockTupleHelpers.java create mode 100644 src/jvm/test/storm/starter/tools/NthLastModifiedTimeTrackerTest.java create mode 100644 src/jvm/test/storm/starter/tools/RankableObjectWithFieldsTest.java create mode 100644 src/jvm/test/storm/starter/tools/RankingsTest.java create mode 100644 src/jvm/test/storm/starter/tools/SlidingWindowCounterTest.java create mode 100644 src/jvm/test/storm/starter/tools/SlotBasedCounterTest.java diff --git a/m2-pom.xml b/m2-pom.xml index 443e1a6872..6afcbc77a4 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -35,6 +35,24 @@ 3.8.1 test + + org.testng + testng + 6.8 + test + + + org.mockito + mockito-all + 1.9.0 + test + + + org.easytesting + fest-assert-core + 2.0M8 + test + storm storm @@ -57,10 +75,21 @@ json-simple 1.1 + + com.google.guava + guava + 13.0.1 + + + commons-collections + commons-collections + 3.2.1 + - src/jvm + src/jvm/main + src/jvm/test ${basedir}/multilang diff --git a/src/jvm/storm/starter/BasicDRPCTopology.java b/src/jvm/main/storm/starter/BasicDRPCTopology.java similarity index 100% rename from src/jvm/storm/starter/BasicDRPCTopology.java rename to src/jvm/main/storm/starter/BasicDRPCTopology.java diff --git a/src/jvm/storm/starter/ExclamationTopology.java b/src/jvm/main/storm/starter/ExclamationTopology.java similarity index 100% rename from src/jvm/storm/starter/ExclamationTopology.java rename to src/jvm/main/storm/starter/ExclamationTopology.java diff --git a/src/jvm/storm/starter/ManualDRPC.java b/src/jvm/main/storm/starter/ManualDRPC.java similarity index 100% rename from src/jvm/storm/starter/ManualDRPC.java rename to src/jvm/main/storm/starter/ManualDRPC.java diff --git a/src/jvm/storm/starter/PrintSampleStream.java b/src/jvm/main/storm/starter/PrintSampleStream.java similarity index 100% rename from src/jvm/storm/starter/PrintSampleStream.java rename to src/jvm/main/storm/starter/PrintSampleStream.java diff --git a/src/jvm/storm/starter/ReachTopology.java b/src/jvm/main/storm/starter/ReachTopology.java similarity index 100% rename from src/jvm/storm/starter/ReachTopology.java rename to src/jvm/main/storm/starter/ReachTopology.java diff --git a/src/jvm/main/storm/starter/RollingTopWords.java b/src/jvm/main/storm/starter/RollingTopWords.java new file mode 100644 index 0000000000..ef016a4daf --- /dev/null +++ b/src/jvm/main/storm/starter/RollingTopWords.java @@ -0,0 +1,61 @@ +package storm.starter; + +import storm.starter.bolt.IntermediateRankingsBolt; +import storm.starter.bolt.RollingCountBolt; +import storm.starter.bolt.TotalRankingsBolt; +import storm.starter.util.StormRunner; +import backtype.storm.Config; +import backtype.storm.testing.TestWordSpout; +import backtype.storm.topology.TopologyBuilder; +import backtype.storm.tuple.Fields; + +/** + * This topology does a continuous computation of the top N words that the topology has seen in terms of cardinality. + * The top N computation is done in a completely scalable way, and a similar approach could be used to compute things + * like trending topics or trending images on Twitter. + */ +public class RollingTopWords { + + private static final int DEFAULT_RUNTIME_IN_SECONDS = 60; + private static final int TOP_N = 5; + + private final TopologyBuilder builder; + private final String topologyName; + private final Config topologyConfig; + private final int runtimeInSeconds; + + public RollingTopWords() throws InterruptedException { + builder = new TopologyBuilder(); + topologyName = "slidingWindowCounts"; + topologyConfig = createTopologyConfiguration(); + runtimeInSeconds = DEFAULT_RUNTIME_IN_SECONDS; + + wireTopology(); + } + + private static Config createTopologyConfiguration() { + Config conf = new Config(); + conf.setDebug(true); + return conf; + } + + private void wireTopology() throws InterruptedException { + String spoutId = "wordGenerator"; + String counterId = "counter"; + String intermediateRankerId = "intermediateRanker"; + String totalRankerId = "finalRanker"; + builder.setSpout(spoutId, new TestWordSpout(), 5); + builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word")); + builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId, + new Fields("obj")); + builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId); + } + + public void run() throws InterruptedException { + StormRunner.runTopologyLocally(builder.createTopology(), topologyName, topologyConfig, runtimeInSeconds); + } + + public static void main(String[] args) throws Exception { + new RollingTopWords().run(); + } +} diff --git a/src/jvm/storm/starter/SingleJoinExample.java b/src/jvm/main/storm/starter/SingleJoinExample.java similarity index 100% rename from src/jvm/storm/starter/SingleJoinExample.java rename to src/jvm/main/storm/starter/SingleJoinExample.java diff --git a/src/jvm/storm/starter/TransactionalGlobalCount.java b/src/jvm/main/storm/starter/TransactionalGlobalCount.java similarity index 100% rename from src/jvm/storm/starter/TransactionalGlobalCount.java rename to src/jvm/main/storm/starter/TransactionalGlobalCount.java diff --git a/src/jvm/storm/starter/TransactionalWords.java b/src/jvm/main/storm/starter/TransactionalWords.java similarity index 100% rename from src/jvm/storm/starter/TransactionalWords.java rename to src/jvm/main/storm/starter/TransactionalWords.java diff --git a/src/jvm/storm/starter/WordCountTopology.java b/src/jvm/main/storm/starter/WordCountTopology.java similarity index 100% rename from src/jvm/storm/starter/WordCountTopology.java rename to src/jvm/main/storm/starter/WordCountTopology.java diff --git a/src/jvm/main/storm/starter/bolt/AbstractRankerBolt.java b/src/jvm/main/storm/starter/bolt/AbstractRankerBolt.java new file mode 100644 index 0000000000..89e26551f6 --- /dev/null +++ b/src/jvm/main/storm/starter/bolt/AbstractRankerBolt.java @@ -0,0 +1,95 @@ +package storm.starter.bolt; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.log4j.Logger; + +import storm.starter.tools.Rankings; +import storm.starter.util.TupleHelpers; +import backtype.storm.Config; +import backtype.storm.topology.BasicOutputCollector; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseBasicBolt; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; + +/** + * This abstract bolt provides the basic behavior of bolts that rank objects according to their count. + * + * It uses a template method design pattern for {@link AbstractRankerBolt#execute(Tuple, BasicOutputCollector)} to allow + * actual bolt implementations to specify how incoming tuples are processed, i.e. how the objects embedded within those + * tuples are retrieved and counted. + * + */ +public abstract class AbstractRankerBolt extends BaseBasicBolt { + + private static final long serialVersionUID = 4931640198501530202L; + private static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS = 2; + private static final int DEFAULT_COUNT = 10; + + private final int emitFrequencyInSeconds; + private final int count; + private final Rankings rankings; + + public AbstractRankerBolt() { + this(DEFAULT_COUNT, DEFAULT_EMIT_FREQUENCY_IN_SECONDS); + } + + public AbstractRankerBolt(int topN) { + this(topN, DEFAULT_EMIT_FREQUENCY_IN_SECONDS); + } + + public AbstractRankerBolt(int topN, int emitFrequencyInSeconds) { + if (topN < 1) { + throw new IllegalArgumentException("topN must be >= 1 (you requested " + topN + ")"); + } + if (emitFrequencyInSeconds < 1) { + throw new IllegalArgumentException("The emit frequency must be >= 1 seconds (you requested " + + emitFrequencyInSeconds + " seconds)"); + } + count = topN; + this.emitFrequencyInSeconds = emitFrequencyInSeconds; + rankings = new Rankings(count); + } + + protected Rankings getRankings() { + return rankings; + } + + /** + * This method functions as a template method (design pattern). + */ + @Override + public final void execute(Tuple tuple, BasicOutputCollector collector) { + if (TupleHelpers.isTickTuple(tuple)) { + getLogger().info("Received tick tuple, triggering emit of current rankings"); + emitRankings(collector); + } + else { + updateRankingsWithTuple(tuple); + } + } + + abstract void updateRankingsWithTuple(Tuple tuple); + + private void emitRankings(BasicOutputCollector collector) { + collector.emit(new Values(rankings)); + getLogger().info("Rankings: " + rankings); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("rankings")); + } + + @Override + public Map getComponentConfiguration() { + Map conf = new HashMap(); + conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, emitFrequencyInSeconds); + return conf; + } + + abstract Logger getLogger(); +} diff --git a/src/jvm/main/storm/starter/bolt/IntermediateRankingsBolt.java b/src/jvm/main/storm/starter/bolt/IntermediateRankingsBolt.java new file mode 100644 index 0000000000..161e67d0ff --- /dev/null +++ b/src/jvm/main/storm/starter/bolt/IntermediateRankingsBolt.java @@ -0,0 +1,43 @@ +package storm.starter.bolt; + +import org.apache.log4j.Logger; + +import storm.starter.tools.Rankable; +import storm.starter.tools.RankableObjectWithFields; +import backtype.storm.tuple.Tuple; + +/** + * This bolt ranks incoming objects by their count. + * + * It assumes the input tuples to adhere to the following format: (object, object_count, additionalField1, + * additionalField2, ..., additionalFieldN). + * + */ +public final class IntermediateRankingsBolt extends AbstractRankerBolt { + + private static final long serialVersionUID = -1369800530256637409L; + private static final Logger LOG = Logger.getLogger(IntermediateRankingsBolt.class); + + public IntermediateRankingsBolt() { + super(); + } + + public IntermediateRankingsBolt(int topN) { + super(topN); + } + + public IntermediateRankingsBolt(int topN, int emitFrequencyInSeconds) { + super(topN, emitFrequencyInSeconds); + } + + @Override + void updateRankingsWithTuple(Tuple tuple) { + Rankable rankable = RankableObjectWithFields.from(tuple); + super.getRankings().updateWith(rankable); + } + + @Override + Logger getLogger() { + return LOG; + } +} diff --git a/src/jvm/storm/starter/bolt/PrinterBolt.java b/src/jvm/main/storm/starter/bolt/PrinterBolt.java similarity index 100% rename from src/jvm/storm/starter/bolt/PrinterBolt.java rename to src/jvm/main/storm/starter/bolt/PrinterBolt.java diff --git a/src/jvm/main/storm/starter/bolt/RollingCountBolt.java b/src/jvm/main/storm/starter/bolt/RollingCountBolt.java new file mode 100644 index 0000000000..848fedb14a --- /dev/null +++ b/src/jvm/main/storm/starter/bolt/RollingCountBolt.java @@ -0,0 +1,126 @@ +package storm.starter.bolt; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.log4j.Logger; + +import storm.starter.tools.NthLastModifiedTimeTracker; +import storm.starter.tools.SlidingWindowCounter; +import storm.starter.util.TupleHelpers; +import backtype.storm.Config; +import backtype.storm.task.OutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseRichBolt; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; + +/** + * This bolt performs rolling counts of incoming objects, i.e. sliding window based counting. + * + * The bolt is configured by two parameters, the length of the sliding window in seconds (which influences the output + * data of the bolt, i.e. how it will count objects) and the emit frequency in seconds (which influences how often the + * bolt will output the latest window counts). For instance, if the window length is set to an equivalent of five + * minutes and the emit frequency to one minute, then the bolt will output the latest five-minute sliding window every + * minute. + * + * The bolt emits a rolling count tuple per object, consisting of the object itself, its latest rolling count, and the + * actual duration of the sliding window. The latter is included in case the expected sliding window length (as + * configured by the user) is different from the actual length, e.g. due to high system load. Note that the actual + * window length is tracked and calculated for the window, and not individually for each object within a window. + * + * Note: During the startup phase you will usually observe that the bolt warns you about the actual sliding window + * length being smaller than the expected length. This behavior is expected and is caused by the way the sliding window + * counts are initially "loaded up". You can safely ignore this warning during startup (e.g. you will see this warning + * during the first ~ five minutes of startup time if the window length is set to five minutes). + * + */ +public class RollingCountBolt extends BaseRichBolt { + + private static final long serialVersionUID = 5537727428628598519L; + private static final Logger LOG = Logger.getLogger(RollingCountBolt.class); + private static final int NUM_WINDOW_CHUNKS = 5; + private static final int DEFAULT_SLIDING_WINDOW_IN_SECONDS = NUM_WINDOW_CHUNKS * 60; + private static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS = DEFAULT_SLIDING_WINDOW_IN_SECONDS / NUM_WINDOW_CHUNKS; + private static final String WINDOW_LENGTH_WARNING_TEMPLATE = "Actual window length is %d seconds when it should be %d seconds" + + " (you can safely ignore this warning during the startup phase)"; + + private final SlidingWindowCounter counter; + private final int windowLengthInSeconds; + private final int emitFrequencyInSeconds; + private OutputCollector collector; + private NthLastModifiedTimeTracker lastModifiedTracker; + + public RollingCountBolt() { + this(DEFAULT_SLIDING_WINDOW_IN_SECONDS, DEFAULT_EMIT_FREQUENCY_IN_SECONDS); + } + + public RollingCountBolt(int windowLengthInSeconds, int emitFrequencyInSeconds) { + this.windowLengthInSeconds = windowLengthInSeconds; + this.emitFrequencyInSeconds = emitFrequencyInSeconds; + counter = new SlidingWindowCounter(deriveNumWindowChunksFrom(this.windowLengthInSeconds, + this.emitFrequencyInSeconds)); + } + + private int deriveNumWindowChunksFrom(int windowLengthInSeconds, int windowUpdateFrequencyInSeconds) { + return windowLengthInSeconds / windowUpdateFrequencyInSeconds; + } + + @SuppressWarnings("rawtypes") + @Override + public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { + this.collector = collector; + lastModifiedTracker = new NthLastModifiedTimeTracker(deriveNumWindowChunksFrom(this.windowLengthInSeconds, + this.emitFrequencyInSeconds)); + } + + @Override + public void execute(Tuple tuple) { + if (TupleHelpers.isTickTuple(tuple)) { + LOG.info("Received tick tuple, triggering emit of current window counts"); + emitCurrentWindowCounts(); + } + else { + countObjAndAck(tuple); + } + } + + private void emitCurrentWindowCounts() { + Map counts = counter.getCountsThenAdvanceWindow(); + int actualWindowLengthInSeconds = lastModifiedTracker.secondsSinceOldestModification(); + lastModifiedTracker.markAsModified(); + if (actualWindowLengthInSeconds != windowLengthInSeconds) { + LOG.warn(String.format(WINDOW_LENGTH_WARNING_TEMPLATE, actualWindowLengthInSeconds, windowLengthInSeconds)); + } + emit(counts, actualWindowLengthInSeconds); + } + + private void emit(Map counts, int actualWindowLengthInSeconds) { + for (Entry entry : counts.entrySet()) { + Object obj = entry.getKey(); + Long count = entry.getValue(); + collector.emit(new Values(obj, count, actualWindowLengthInSeconds)); + } + } + + private void countObjAndAck(Tuple tuple) { + Object obj = tuple.getValue(0); + counter.incrementCount(obj); + collector.ack(tuple); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("obj", "count", "actualWindowLengthInSeconds")); + } + + @Override + public Map getComponentConfiguration() { + Map conf = new HashMap(); + conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, emitFrequencyInSeconds); + return conf; + } +} diff --git a/src/jvm/storm/starter/bolt/SingleJoinBolt.java b/src/jvm/main/storm/starter/bolt/SingleJoinBolt.java similarity index 100% rename from src/jvm/storm/starter/bolt/SingleJoinBolt.java rename to src/jvm/main/storm/starter/bolt/SingleJoinBolt.java diff --git a/src/jvm/main/storm/starter/bolt/TotalRankingsBolt.java b/src/jvm/main/storm/starter/bolt/TotalRankingsBolt.java new file mode 100644 index 0000000000..84ff69e5d6 --- /dev/null +++ b/src/jvm/main/storm/starter/bolt/TotalRankingsBolt.java @@ -0,0 +1,43 @@ +package storm.starter.bolt; + +import org.apache.log4j.Logger; + +import storm.starter.tools.Rankings; +import backtype.storm.tuple.Tuple; + +/** + * This bolt merges incoming {@link Rankings}. + * + * It can be used to merge intermediate rankings generated by {@link IntermediateRankingsBolt} into a final, + * consolidated ranking. To do so, configure this bolt with a globalGrouping on {@link IntermediateRankingsBolt}. + * + */ +public final class TotalRankingsBolt extends AbstractRankerBolt { + + private static final long serialVersionUID = -8447525895532302198L; + private static final Logger LOG = Logger.getLogger(TotalRankingsBolt.class); + + public TotalRankingsBolt() { + super(); + } + + public TotalRankingsBolt(int topN) { + super(topN); + } + + public TotalRankingsBolt(int topN, int emitFrequencyInSeconds) { + super(topN, emitFrequencyInSeconds); + } + + @Override + void updateRankingsWithTuple(Tuple tuple) { + Rankings rankingsToBeMerged = (Rankings) tuple.getValue(0); + super.getRankings().updateWith(rankingsToBeMerged); + } + + @Override + Logger getLogger() { + return LOG; + } + +} diff --git a/src/jvm/storm/starter/spout/RandomSentenceSpout.java b/src/jvm/main/storm/starter/spout/RandomSentenceSpout.java similarity index 100% rename from src/jvm/storm/starter/spout/RandomSentenceSpout.java rename to src/jvm/main/storm/starter/spout/RandomSentenceSpout.java diff --git a/src/jvm/storm/starter/spout/TwitterSampleSpout.java b/src/jvm/main/storm/starter/spout/TwitterSampleSpout.java similarity index 100% rename from src/jvm/storm/starter/spout/TwitterSampleSpout.java rename to src/jvm/main/storm/starter/spout/TwitterSampleSpout.java diff --git a/src/jvm/main/storm/starter/tools/NthLastModifiedTimeTracker.java b/src/jvm/main/storm/starter/tools/NthLastModifiedTimeTracker.java new file mode 100644 index 0000000000..4b85cd27f8 --- /dev/null +++ b/src/jvm/main/storm/starter/tools/NthLastModifiedTimeTracker.java @@ -0,0 +1,55 @@ +package storm.starter.tools; + +import org.apache.commons.collections.buffer.CircularFifoBuffer; + +import backtype.storm.utils.Time; + +/** + * This class tracks the time-since-last-modify of a "thing" in a rolling fashion. + * + * For example, create a 5-slot tracker to track the five most recent time-since-last-modify. + * + * You must manually "mark" that the "something" that you want to track -- in terms of modification times -- has just + * been modified. + * + */ +public class NthLastModifiedTimeTracker { + + private static final int MILLIS_IN_SEC = 1000; + + private final CircularFifoBuffer lastModifiedTimesMillis; + + public NthLastModifiedTimeTracker(int numTimesToTrack) { + if (numTimesToTrack < 1) { + throw new IllegalArgumentException("numTimesToTrack must be greater than zero (you requested " + + numTimesToTrack + ")"); + } + lastModifiedTimesMillis = new CircularFifoBuffer(numTimesToTrack); + initLastModifiedTimesMillis(); + } + + private void initLastModifiedTimesMillis() { + long nowCached = now(); + for (int i = 0; i < lastModifiedTimesMillis.maxSize(); i++) { + lastModifiedTimesMillis.add(Long.valueOf(nowCached)); + } + } + + private long now() { + return Time.currentTimeMillis(); + } + + public int secondsSinceOldestModification() { + long modifiedTimeMillis = ((Long) lastModifiedTimesMillis.get()).longValue(); + return (int) ((now() - modifiedTimeMillis) / MILLIS_IN_SEC); + } + + public void markAsModified() { + updateLastModifiedTime(); + } + + private void updateLastModifiedTime() { + lastModifiedTimesMillis.add(now()); + } + +} diff --git a/src/jvm/main/storm/starter/tools/Rankable.java b/src/jvm/main/storm/starter/tools/Rankable.java new file mode 100644 index 0000000000..03f742fa03 --- /dev/null +++ b/src/jvm/main/storm/starter/tools/Rankable.java @@ -0,0 +1,9 @@ +package storm.starter.tools; + +public interface Rankable extends Comparable { + + Object getObject(); + + long getCount(); + +} diff --git a/src/jvm/main/storm/starter/tools/RankableObjectWithFields.java b/src/jvm/main/storm/starter/tools/RankableObjectWithFields.java new file mode 100644 index 0000000000..43be75a77f --- /dev/null +++ b/src/jvm/main/storm/starter/tools/RankableObjectWithFields.java @@ -0,0 +1,117 @@ +package storm.starter.tools; + +import java.util.List; + +import backtype.storm.tuple.Tuple; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +/** + * This class wraps an objects and its associated count, including any additional data fields. + * + * This class can be used, for instance, to track the number of occurrences of an object in a Storm topology. + * + */ +public class RankableObjectWithFields implements Rankable { + + private static final String toStringSeparator = "|"; + + private final Object obj; + private final long count; + private final ImmutableList fields; + + public RankableObjectWithFields(Object obj, long count, Object... otherFields) { + if (obj == null) { + throw new IllegalArgumentException("The object must not be null"); + } + if (count < 0) { + throw new IllegalArgumentException("The count must be >= 0"); + } + this.obj = obj; + this.count = count; + fields = ImmutableList.copyOf(otherFields); + + } + + /** + * Construct a new instance based on the provided {@link Tuple}. + * + * This method expects the object to be ranked in the first field (index 0) of the provided tuple, and the number of + * occurrences of the object (its count) in the second field (index 1). Any further fields in the tuple will be + * extracted and tracked, too. These fields can be accessed via {@link RankableObjectWithFields#getFields()}. + * + * @param tuple + * @return + */ + public static RankableObjectWithFields from(Tuple tuple) { + List otherFields = Lists.newArrayList(tuple.getValues()); + Object obj = otherFields.remove(0); + Long count = (Long) otherFields.remove(0); + return new RankableObjectWithFields(obj, count, otherFields.toArray()); + } + + public Object getObject() { + return obj; + } + + public long getCount() { + return count; + } + + /** + * @return an immutable list of any additional data fields of the object (may be empty but will never be null) + */ + public List getFields() { + return fields; + } + + @Override + public int compareTo(Rankable other) { + long delta = this.getCount() - other.getCount(); + if (delta > 0) { + return 1; + } + else if (delta < 0) { + return -1; + } + else { + return 0; + } + } + + @Override + public boolean equals(Object o) { + if (this == obj) { + return true; + } + if (!(o instanceof RankableObjectWithFields)) { + return false; + } + RankableObjectWithFields other = (RankableObjectWithFields) o; + return obj.equals(other.obj) && count == other.count; + } + + @Override + public int hashCode() { + int result = 17; + int countHash = (int) (count ^ (count >>> 32)); + result = 31 * result + countHash; + result = 31 * result + obj.hashCode(); + return result; + } + + public String toString() { + StringBuffer buf = new StringBuffer(); + buf.append("["); + buf.append(obj); + buf.append(toStringSeparator); + buf.append(count); + for (Object field : fields) { + buf.append(toStringSeparator); + buf.append(field); + } + buf.append("]"); + return buf.toString(); + } +} diff --git a/src/jvm/main/storm/starter/tools/Rankings.java b/src/jvm/main/storm/starter/tools/Rankings.java new file mode 100644 index 0000000000..4eee168835 --- /dev/null +++ b/src/jvm/main/storm/starter/tools/Rankings.java @@ -0,0 +1,93 @@ +package storm.starter.tools; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; + +import com.google.common.collect.Lists; + +public class Rankings implements Serializable { + + private static final long serialVersionUID = -1549827195410578903L; + private static final int DEFAULT_COUNT = 10; + + private final int maxSize; + private final List rankedItems = Lists.newArrayList(); + + public Rankings() { + this(DEFAULT_COUNT); + } + + public Rankings(int topN) { + if (topN < 1) { + throw new IllegalArgumentException("topN must be >= 1"); + } + maxSize = topN; + } + + /** + * @return the maximum possible number (size) of ranked objects this instance can hold + */ + public int maxSize() { + return maxSize; + } + + /** + * @return the number (size) of ranked objects this instance is currently holding + */ + public int size() { + return rankedItems.size(); + } + + public List getRankings() { + return Lists.newArrayList(rankedItems); + } + + public void updateWith(Rankings other) { + for (Rankable r : other.getRankings()) { + updateWith(r); + } + } + + public void updateWith(Rankable r) { + addOrReplace(r); + rerank(); + shrinkRankingsIfNeeded(); + } + + private void addOrReplace(Rankable r) { + Integer rank = findRankOf(r); + if (rank != null) { + rankedItems.set(rank, r); + } + else { + rankedItems.add(r); + } + } + + private Integer findRankOf(Rankable r) { + Object tag = r.getObject(); + for (int rank = 0; rank < rankedItems.size(); rank++) { + Object cur = rankedItems.get(rank).getObject(); + if (cur.equals(tag)) { + return rank; + } + } + return null; + } + + private void rerank() { + Collections.sort(rankedItems); + Collections.reverse(rankedItems); + } + + private void shrinkRankingsIfNeeded() { + if (rankedItems.size() > maxSize) { + rankedItems.remove(maxSize); + } + } + + public String toString() { + return rankedItems.toString(); + } +} diff --git a/src/jvm/main/storm/starter/tools/SlidingWindowCounter.java b/src/jvm/main/storm/starter/tools/SlidingWindowCounter.java new file mode 100644 index 0000000000..9ccd62ce11 --- /dev/null +++ b/src/jvm/main/storm/starter/tools/SlidingWindowCounter.java @@ -0,0 +1,102 @@ +package storm.starter.tools; + +import java.io.Serializable; +import java.util.Map; + +/** + * This class counts objects in a sliding window fashion. + * + * It is designed 1) to give multiple "producer" threads write access to the counter, i.e. being able to increment + * counts of objects, and 2) to give a single "consumer" thread (e.g. {@link PeriodicSlidingWindowCounter}) read access + * to the counter. Whenever the consumer thread performs a read operation, this class will advance the head slot of the + * sliding window counter. This means that the consumer thread indirectly controls where writes of the producer threads + * will go to. Also, by itself this class will not advance the head slot. + * + * A note for analyzing data based on a sliding window count: During the initial windowLengthInSlots + * iterations, this sliding window counter will always return object counts that are equal or greater than in the + * previous iteration. This is the effect of the counter "loading up" at the very start of its existence. Conceptually, + * this is the desired behavior. + * + * To give an example, using a counter with 5 slots which for the sake of this example represent 1 minute of time each: + * + * 
+ * {@code
+ * Sliding window counts of an object X over time
+ * 
+ * Minute (timeline):
+ * 1    2   3   4   5   6   7   8
+ * 
+ * Observed counts per minute:
+ * 1    1   1   1   0   0   0   0
+ * 
+ * Counts returned by counter:
+ * 1    2   3   4   4   3   2   1
+ * }
+ *
+ * + * As you can see in this example, for the first windowLengthInSlots (here: the first five minutes) the + * counter will always return counts equal or greater than in the previous iteration (1, 2, 3, 4, 4). This initial load + * effect needs to be accounted for whenever you want to perform analyses such as trending topics; otherwise your + * analysis algorithm might falsely identify the object to be trending as the counter seems to observe continuously + * increasing counts. Also, note that during the initial load phase every object will exhibit increasing + * counts. + * + * On a high-level, the counter exhibits the following behavior: If you asked the example counter after two minutes, + * "how often did you count the object during the past five minutes?", then it should reply + * "I have counted it 2 times in the past five minutes", implying that it can only account for the last two of those + * five minutes because the counter was not running before that time. + * + * @param + * The type of those objects we want to count. + */ +public final class SlidingWindowCounter implements Serializable { + + private static final long serialVersionUID = -2645063988768785810L; + + private SlotBasedCounter objCounter; + private int headSlot; + private int tailSlot; + private int windowLengthInSlots; + + public SlidingWindowCounter(int windowLengthInSlots) { + if (windowLengthInSlots < 2) { + throw new IllegalArgumentException("Window length in slots must be at least two (you requested " + + windowLengthInSlots + ")"); + } + this.windowLengthInSlots = windowLengthInSlots; + this.objCounter = new SlotBasedCounter(this.windowLengthInSlots); + + this.headSlot = 0; + this.tailSlot = slotAfter(headSlot); + } + + public void incrementCount(T obj) { + objCounter.incrementCount(obj, headSlot); + } + + /** + * Return the current (total) counts of all tracked objects, then advance the window. + * + * Whenever this method is called, we consider the counts of the current sliding window to be available to and + * successfully processed "upstream" (i.e. by the caller). Knowing this we will start counting any subsequent + * objects within the next "chunk" of the sliding window. + * + * @return + */ + public Map getCountsThenAdvanceWindow() { + Map counts = objCounter.getCounts(); + objCounter.wipeSlot(tailSlot); + advanceHead(); + return counts; + } + + private void advanceHead() { + headSlot = tailSlot; + tailSlot = slotAfter(tailSlot); + } + + private int slotAfter(int slot) { + return (slot + 1) % windowLengthInSlots; + } + +} diff --git a/src/jvm/main/storm/starter/tools/SlotBasedCounter.java b/src/jvm/main/storm/starter/tools/SlotBasedCounter.java new file mode 100644 index 0000000000..7826b51f3b --- /dev/null +++ b/src/jvm/main/storm/starter/tools/SlotBasedCounter.java @@ -0,0 +1,98 @@ +package storm.starter.tools; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * This class provides per-slot counts of the occurrences of objects. + * + * It can be used, for instance, as a building block for implementing sliding window counting of objects. + * + * @param + * The type of those objects we want to count. + */ +public final class SlotBasedCounter implements Serializable { + + private static final long serialVersionUID = 4858185737378394432L; + + private final Map objToCounts = new HashMap(); + private final int numSlots; + + public SlotBasedCounter(int numSlots) { + if (numSlots <= 0) { + throw new IllegalArgumentException("Number of slots must be greater than zero (you requested " + numSlots + + ")"); + } + this.numSlots = numSlots; + } + + public void incrementCount(T obj, int slot) { + long[] counts = objToCounts.get(obj); + if (counts == null) { + counts = new long[this.numSlots]; + objToCounts.put(obj, counts); + } + counts[slot]++; + } + + public long getCount(T obj, int slot) { + long[] counts = objToCounts.get(obj); + if (counts == null) { + return 0; + } + else { + return counts[slot]; + } + } + + public Map getCounts() { + Map result = new HashMap(); + for (T obj : objToCounts.keySet()) { + result.put(obj, computeTotalCount(obj)); + } + return result; + } + + private long computeTotalCount(T obj) { + long[] curr = objToCounts.get(obj); + long total = 0; + for (long l : curr) { + total += l; + } + return total; + } + + /** + * Reset the slot count of any tracked objects to zero for the given slot. + * + * Implementation detail: As an optimization this method will also remove any object from the counter whose + * total count is zero after the wipe of the slot (to free up memory). + * + * @param slot + */ + public void wipeSlot(int slot) { + Set objToBeRemoved = new HashSet(); + for (T obj : objToCounts.keySet()) { + resetSlotCountToZero(obj, slot); + if (shouldBeRemovedFromCounter(obj)) { + objToBeRemoved.add(obj); + } + } + for (T obj : objToBeRemoved) { + objToCounts.remove(obj); + } + } + + private void resetSlotCountToZero(T obj, int slot) { + long[] counts = objToCounts.get(obj); + counts[slot] = 0; + } + + private boolean shouldBeRemovedFromCounter(T obj) { + return computeTotalCount(obj) == 0; + } + +} diff --git a/src/jvm/storm/starter/trident/TridentReach.java b/src/jvm/main/storm/starter/trident/TridentReach.java similarity index 100% rename from src/jvm/storm/starter/trident/TridentReach.java rename to src/jvm/main/storm/starter/trident/TridentReach.java diff --git a/src/jvm/storm/starter/trident/TridentWordCount.java b/src/jvm/main/storm/starter/trident/TridentWordCount.java similarity index 100% rename from src/jvm/storm/starter/trident/TridentWordCount.java rename to src/jvm/main/storm/starter/trident/TridentWordCount.java diff --git a/src/jvm/main/storm/starter/util/StormRunner.java b/src/jvm/main/storm/starter/util/StormRunner.java new file mode 100644 index 0000000000..09b40628cc --- /dev/null +++ b/src/jvm/main/storm/starter/util/StormRunner.java @@ -0,0 +1,22 @@ +package storm.starter.util; + +import backtype.storm.Config; +import backtype.storm.LocalCluster; +import backtype.storm.generated.StormTopology; + +public final class StormRunner { + + private static final int MILLIS_IN_SEC = 1000; + + private StormRunner() { + } + + public static void runTopologyLocally(StormTopology topology, String topologyName, Config conf, int runtimeInSeconds) + throws InterruptedException { + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology(topologyName, conf, topology); + Thread.sleep((long) runtimeInSeconds * MILLIS_IN_SEC); + cluster.killTopology(topologyName); + cluster.shutdown(); + } +} diff --git a/src/jvm/main/storm/starter/util/TupleHelpers.java b/src/jvm/main/storm/starter/util/TupleHelpers.java new file mode 100644 index 0000000000..58693d77ad --- /dev/null +++ b/src/jvm/main/storm/starter/util/TupleHelpers.java @@ -0,0 +1,16 @@ +package storm.starter.util; + +import backtype.storm.Constants; +import backtype.storm.tuple.Tuple; + +public final class TupleHelpers { + + private TupleHelpers() { + } + + public static boolean isTickTuple(Tuple tuple) { + return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID) + && tuple.getSourceStreamId().equals(Constants.SYSTEM_TICK_STREAM_ID); + } + +} diff --git a/src/jvm/storm/starter/RollingTopWords.java b/src/jvm/storm/starter/RollingTopWords.java deleted file mode 100644 index b105c5a700..0000000000 --- a/src/jvm/storm/starter/RollingTopWords.java +++ /dev/null @@ -1,48 +0,0 @@ -package storm.starter; - -import backtype.storm.Config; -import backtype.storm.LocalCluster; -import backtype.storm.testing.TestWordSpout; -import backtype.storm.topology.TopologyBuilder; -import backtype.storm.tuple.Fields; -import storm.starter.bolt.MergeObjects; -import storm.starter.bolt.RankObjects; -import storm.starter.bolt.RollingCountObjects; - - -/** - * This topology does a continuous computation of the top N words that the topology has seen - * in terms of cardinality. The top N computation is done in a completely scalable way, and - * a similar approach could be used to compute things like trending topics or trending images - * on Twitter. - */ -public class RollingTopWords { - - public static void main(String[] args) throws Exception { - - final int TOP_N = 3; - - TopologyBuilder builder = new TopologyBuilder(); - - builder.setSpout("word", new TestWordSpout(), 5); - - builder.setBolt("count", new RollingCountObjects(60, 10), 4) - .fieldsGrouping("word", new Fields("word")); - builder.setBolt("rank", new RankObjects(TOP_N), 4) - .fieldsGrouping("count", new Fields("obj")); - builder.setBolt("merge", new MergeObjects(TOP_N)) - .globalGrouping("rank"); - - - - Config conf = new Config(); - conf.setDebug(true); - - LocalCluster cluster = new LocalCluster(); - cluster.submitTopology("rolling-demo", conf, builder.createTopology()); - Thread.sleep(10000); - - cluster.shutdown(); - - } -} diff --git a/src/jvm/storm/starter/bolt/MergeObjects.java b/src/jvm/storm/starter/bolt/MergeObjects.java deleted file mode 100644 index a536aeb887..0000000000 --- a/src/jvm/storm/starter/bolt/MergeObjects.java +++ /dev/null @@ -1,83 +0,0 @@ -package storm.starter.bolt; - -import org.apache.log4j.Logger; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.BasicOutputCollector; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseBasicBolt; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - -public class MergeObjects extends BaseBasicBolt { - public static Logger LOG = Logger.getLogger(MergeObjects.class); - - private List _rankings = new ArrayList(); - int _count = 10; - Long _lastTime; - - public MergeObjects(int n) { - _count = n; - } - - private int _compare(List one, List two) { - long valueOne = (Long) one.get(1); - long valueTwo = (Long) two.get(1); - long delta = valueTwo - valueOne; - if(delta > 0) { - return 1; - } else if (delta < 0) { - return -1; - } else { - return 0; - } - } - - private Integer _find(Object tag) { - for(int i = 0; i < _rankings.size(); ++i) { - Object cur = _rankings.get(i).get(0); - if (cur.equals(tag)) { - return i; - } - } - return null; - } - - public void execute(Tuple tuple, BasicOutputCollector collector) { - List merging = (List) tuple.getValue(0); - for(List pair : merging) { - Integer existingIndex = _find(pair.get(0)); - if (null != existingIndex) { - _rankings.set(existingIndex, pair); - } else { - _rankings.add(pair); - } - - Collections.sort(_rankings, new Comparator() { - public int compare(List o1, List o2) { - return _compare(o1, o2); - } - }); - - if (_rankings.size() > _count) { - _rankings.subList(_count, _rankings.size()).clear(); - } - } - - long currentTime = System.currentTimeMillis(); - if(_lastTime==null || currentTime >= _lastTime + 2000) { - collector.emit(new Values(new ArrayList(_rankings))); - LOG.info("Rankings: " + _rankings); - _lastTime = currentTime; - } - } - - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("list")); - } -} diff --git a/src/jvm/storm/starter/bolt/RankObjects.java b/src/jvm/storm/starter/bolt/RankObjects.java deleted file mode 100644 index 95455db9b8..0000000000 --- a/src/jvm/storm/starter/bolt/RankObjects.java +++ /dev/null @@ -1,76 +0,0 @@ -package storm.starter.bolt; - -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.BasicOutputCollector; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseBasicBolt; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - - -public class RankObjects extends BaseBasicBolt { - List _rankings = new ArrayList(); - - int _count; - Long _lastTime = null; - - public RankObjects(int n) { - _count = n; - } - - private int _compare(List one, List two) { - long valueOne = (Long) one.get(1); - long valueTwo = (Long) two.get(1); - long delta = valueTwo - valueOne; - if(delta > 0) { - return 1; - } else if (delta < 0) { - return -1; - } else { - return 0; - } - } - - private Integer _find(Object tag) { - for(int i = 0; i < _rankings.size(); ++i) { - Object cur = _rankings.get(i).get(0); - if (cur.equals(tag)) { - return i; - } - } - return null; - } - - public void execute(Tuple tuple, BasicOutputCollector collector) { - Object tag = tuple.getValue(0); - Integer existingIndex = _find(tag); - if (null != existingIndex) { - _rankings.set(existingIndex, tuple.getValues()); - } else { - _rankings.add(tuple.getValues()); - } - Collections.sort(_rankings, new Comparator() { - public int compare(List o1, List o2) { - return _compare(o1, o2); - } - }); - if (_rankings.size() > _count) { - _rankings.remove(_count); - } - long currentTime = System.currentTimeMillis(); - if(_lastTime==null || currentTime >= _lastTime + 2000) { - collector.emit(new Values(new ArrayList(_rankings))); - _lastTime = currentTime; - } - } - - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("list")); - } -} diff --git a/src/jvm/storm/starter/bolt/RollingCountObjects.java b/src/jvm/storm/starter/bolt/RollingCountObjects.java deleted file mode 100644 index a0c06f31e0..0000000000 --- a/src/jvm/storm/starter/bolt/RollingCountObjects.java +++ /dev/null @@ -1,110 +0,0 @@ -package storm.starter.bolt; - -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.topology.base.BaseRichBolt; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import backtype.storm.tuple.Values; -import backtype.storm.utils.Utils; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -public class RollingCountObjects extends BaseRichBolt { - - private HashMap _objectCounts = new HashMap(); - private int _numBuckets; - private transient Thread cleaner; - private OutputCollector _collector; - private int _trackMinutes; - - public RollingCountObjects(int numBuckets, int trackMinutes) { - _numBuckets = numBuckets; - _trackMinutes = trackMinutes; - } - - public long totalObjects (Object obj) { - long[] curr = _objectCounts.get(obj); - long total = 0; - for (long l: curr) { - total+=l; - } - return total; - } - - public int currentBucket (int buckets) { - return (currentSecond() / secondsPerBucket(buckets)) % buckets; - } - - public int currentSecond() { - return (int) (System.currentTimeMillis() / 1000); - } - - public int secondsPerBucket(int buckets) { - return (_trackMinutes * 60 / buckets); - } - - public long millisPerBucket(int buckets) { - return (long) secondsPerBucket(buckets) * 1000; - } - - public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { - _collector = collector; - cleaner = new Thread(new Runnable() { - public void run() { - Integer lastBucket = currentBucket(_numBuckets); - - - while(true) { - int currBucket = currentBucket(_numBuckets); - if(currBucket!=lastBucket) { - int bucketToWipe = (currBucket + 1) % _numBuckets; - synchronized(_objectCounts) { - Set objs = new HashSet(_objectCounts.keySet()); - for (Object obj: objs) { - long[] counts = _objectCounts.get(obj); - long currBucketVal = counts[bucketToWipe]; - counts[bucketToWipe] = 0; - long total = totalObjects(obj); - if(currBucketVal!=0) { - _collector.emit(new Values(obj, total)); - } - if(total==0) { - _objectCounts.remove(obj); - } - } - } - lastBucket = currBucket; - } - long delta = millisPerBucket(_numBuckets) - (System.currentTimeMillis() % millisPerBucket(_numBuckets)); - Utils.sleep(delta); - } - } - }); - cleaner.start(); - } - - public void execute(Tuple tuple) { - - Object obj = tuple.getValue(0); - int bucket = currentBucket(_numBuckets); - synchronized(_objectCounts) { - long[] curr = _objectCounts.get(obj); - if(curr==null) { - curr = new long[_numBuckets]; - _objectCounts.put(obj, curr); - } - curr[bucket]++; - _collector.emit(new Values(obj, totalObjects(obj))); - _collector.ack(tuple); - } - } - - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("obj", "count")); - } - -} diff --git a/src/jvm/test/storm/starter/bolt/IntermediateRankingsBoltTest.java b/src/jvm/test/storm/starter/bolt/IntermediateRankingsBoltTest.java new file mode 100644 index 0000000000..c51d0816a7 --- /dev/null +++ b/src/jvm/test/storm/starter/bolt/IntermediateRankingsBoltTest.java @@ -0,0 +1,135 @@ +package storm.starter.bolt; + +import static org.fest.assertions.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +import java.util.Map; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import storm.starter.tools.MockTupleHelpers; +import backtype.storm.Config; +import backtype.storm.topology.BasicOutputCollector; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; + +import com.google.common.collect.Lists; + +public class IntermediateRankingsBoltTest { + + private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id"; + private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id"; + private static final Object ANY_OBJECT = new Object(); + private static final int ANY_TOPN = 10; + private static final long ANY_COUNT = 42; + + private Tuple mockRankableTuple(Object obj, long count) { + Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID); + when(tuple.getValues()).thenReturn(Lists.newArrayList(ANY_OBJECT, ANY_COUNT)); + return tuple; + } + + @DataProvider + public Object[][] illegalTopN() { + return new Object[][] { { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopN") + public void negativeOrZeroTopNShouldThrowIAE(int topN) { + new IntermediateRankingsBolt(topN); + } + + @DataProvider + public Object[][] illegalEmitFrequency() { + return new Object[][] { { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalEmitFrequency") + public void negativeOrZeroEmitFrequencyShouldThrowIAE(int emitFrequencyInSeconds) { + new IntermediateRankingsBolt(ANY_TOPN, emitFrequencyInSeconds); + } + + @DataProvider + public Object[][] legalTopN() { + return new Object[][] { { 1 }, { 2 }, { 3 }, { 20 } }; + } + + @Test(dataProvider = "legalTopN") + public void positiveTopNShouldBeOk(int topN) { + new IntermediateRankingsBolt(topN); + } + + @DataProvider + public Object[][] legalEmitFrequency() { + return new Object[][] { { 1 }, { 2 }, { 3 }, { 20 } }; + } + + @Test(dataProvider = "legalEmitFrequency") + public void positiveEmitFrequencyShouldBeOk(int emitFrequencyInSeconds) { + new IntermediateRankingsBolt(ANY_TOPN, emitFrequencyInSeconds); + } + + @Test + public void shouldEmitSomethingIfTickTupleIsReceived() { + // given + Tuple tickTuple = MockTupleHelpers.mockTickTuple(); + BasicOutputCollector collector = mock(BasicOutputCollector.class); + IntermediateRankingsBolt bolt = new IntermediateRankingsBolt(); + + // when + bolt.execute(tickTuple, collector); + + // then + // verifyZeroInteractions(collector); + verify(collector).emit(any(Values.class)); + } + + @Test + public void shouldEmitNothingIfNormalTupleIsReceived() { + // given + Tuple normalTuple = mockRankableTuple(ANY_OBJECT, ANY_COUNT); + BasicOutputCollector collector = mock(BasicOutputCollector.class); + IntermediateRankingsBolt bolt = new IntermediateRankingsBolt(); + + // when + bolt.execute(normalTuple, collector); + + // then + verifyZeroInteractions(collector); + } + + @Test + public void shouldDeclareOutputFields() { + // given + OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); + IntermediateRankingsBolt bolt = new IntermediateRankingsBolt(); + + // when + bolt.declareOutputFields(declarer); + + // then + verify(declarer, times(1)).declare(any(Fields.class)); + } + + @Test + public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() { + // given + IntermediateRankingsBolt bolt = new IntermediateRankingsBolt(); + + // when + Map componentConfig = bolt.getComponentConfiguration(); + + // then + assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + assertThat(emitFrequencyInSeconds).isGreaterThan(0); + } +} diff --git a/src/jvm/test/storm/starter/bolt/RollingCountBoltTest.java b/src/jvm/test/storm/starter/bolt/RollingCountBoltTest.java new file mode 100644 index 0000000000..ae19041e82 --- /dev/null +++ b/src/jvm/test/storm/starter/bolt/RollingCountBoltTest.java @@ -0,0 +1,101 @@ +package storm.starter.bolt; + +import static org.fest.assertions.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +import java.util.Map; + +import org.testng.annotations.Test; + +import storm.starter.tools.MockTupleHelpers; +import backtype.storm.Config; +import backtype.storm.task.OutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; + +public class RollingCountBoltTest { + + private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id"; + private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id"; + + private Tuple mockNormalTuple(Object obj) { + Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID); + when(tuple.getValue(0)).thenReturn(obj); + return tuple; + } + + @SuppressWarnings("rawtypes") + @Test + public void shouldEmitNothingIfNoObjectHasBeenCountedYetAndTickTupleIsReceived() { + // given + Tuple tickTuple = MockTupleHelpers.mockTickTuple(); + RollingCountBolt bolt = new RollingCountBolt(); + Map conf = mock(Map.class); + TopologyContext context = mock(TopologyContext.class); + OutputCollector collector = mock(OutputCollector.class); + bolt.prepare(conf, context, collector); + + // when + bolt.execute(tickTuple); + + // then + verifyZeroInteractions(collector); + } + + @SuppressWarnings("rawtypes") + @Test + public void shouldEmitSomethingIfAtLeastOneObjectWasCountedAndTickTupleIsReceived() { + // given + Tuple normalTuple = mockNormalTuple(new Object()); + Tuple tickTuple = MockTupleHelpers.mockTickTuple(); + + RollingCountBolt bolt = new RollingCountBolt(); + Map conf = mock(Map.class); + TopologyContext context = mock(TopologyContext.class); + OutputCollector collector = mock(OutputCollector.class); + bolt.prepare(conf, context, collector); + + // when + bolt.execute(normalTuple); + bolt.execute(tickTuple); + + // then + verify(collector).emit(any(Values.class)); + } + + @Test + public void shouldDeclareOutputFields() { + // given + OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); + RollingCountBolt bolt = new RollingCountBolt(); + + // when + bolt.declareOutputFields(declarer); + + // then + verify(declarer, times(1)).declare(any(Fields.class)); + + } + + @Test + public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() { + // given + RollingCountBolt bolt = new RollingCountBolt(); + + // when + Map componentConfig = bolt.getComponentConfiguration(); + + // then + assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + assertThat(emitFrequencyInSeconds).isGreaterThan(0); + } +} diff --git a/src/jvm/test/storm/starter/bolt/TotalRankingsBoltTest.java b/src/jvm/test/storm/starter/bolt/TotalRankingsBoltTest.java new file mode 100644 index 0000000000..69f531950f --- /dev/null +++ b/src/jvm/test/storm/starter/bolt/TotalRankingsBoltTest.java @@ -0,0 +1,135 @@ +package storm.starter.bolt; + +import static org.fest.assertions.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +import java.util.Map; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import storm.starter.tools.MockTupleHelpers; +import storm.starter.tools.Rankings; +import backtype.storm.Config; +import backtype.storm.topology.BasicOutputCollector; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.Values; + +public class TotalRankingsBoltTest { + + private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id"; + private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id"; + private static final Object ANY_OBJECT = new Object(); + private static final int ANY_TOPN = 10; + private static final long ANY_COUNT = 42; + + private Tuple mockRankingsTuple(Object obj, long count) { + Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID); + Rankings rankings = mock(Rankings.class); + when(tuple.getValue(0)).thenReturn(rankings); + return tuple; + } + + @DataProvider + public Object[][] illegalTopN() { + return new Object[][] { { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopN") + public void negativeOrZeroTopNShouldThrowIAE(int topN) { + new TotalRankingsBolt(topN); + } + + @DataProvider + public Object[][] illegalEmitFrequency() { + return new Object[][] { { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalEmitFrequency") + public void negativeOrZeroEmitFrequencyShouldThrowIAE(int emitFrequencyInSeconds) { + new TotalRankingsBolt(ANY_TOPN, emitFrequencyInSeconds); + } + + @DataProvider + public Object[][] legalTopN() { + return new Object[][] { { 1 }, { 2 }, { 3 }, { 20 } }; + } + + @Test(dataProvider = "legalTopN") + public void positiveTopNShouldBeOk(int topN) { + new TotalRankingsBolt(topN); + } + + @DataProvider + public Object[][] legalEmitFrequency() { + return new Object[][] { { 1 }, { 2 }, { 3 }, { 20 } }; + } + + @Test(dataProvider = "legalEmitFrequency") + public void positiveEmitFrequencyShouldBeOk(int emitFrequencyInSeconds) { + new TotalRankingsBolt(ANY_TOPN, emitFrequencyInSeconds); + } + + @Test + public void shouldEmitSomethingIfTickTupleIsReceived() { + // given + Tuple tickTuple = MockTupleHelpers.mockTickTuple(); + BasicOutputCollector collector = mock(BasicOutputCollector.class); + TotalRankingsBolt bolt = new TotalRankingsBolt(); + + // when + bolt.execute(tickTuple, collector); + + // then + // verifyZeroInteractions(collector); + verify(collector).emit(any(Values.class)); + } + + @Test + public void shouldEmitNothingIfNormalTupleIsReceived() { + // given + Tuple normalTuple = mockRankingsTuple(ANY_OBJECT, ANY_COUNT); + BasicOutputCollector collector = mock(BasicOutputCollector.class); + TotalRankingsBolt bolt = new TotalRankingsBolt(); + + // when + bolt.execute(normalTuple, collector); + + // then + verifyZeroInteractions(collector); + } + + @Test + public void shouldDeclareOutputFields() { + // given + OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); + TotalRankingsBolt bolt = new TotalRankingsBolt(); + + // when + bolt.declareOutputFields(declarer); + + // then + verify(declarer, times(1)).declare(any(Fields.class)); + } + + @Test + public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() { + // given + TotalRankingsBolt bolt = new TotalRankingsBolt(); + + // when + Map componentConfig = bolt.getComponentConfiguration(); + + // then + assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + assertThat(emitFrequencyInSeconds).isGreaterThan(0); + } +} diff --git a/src/jvm/test/storm/starter/tools/MockTupleHelpers.java b/src/jvm/test/storm/starter/tools/MockTupleHelpers.java new file mode 100644 index 0000000000..d0a9e419e7 --- /dev/null +++ b/src/jvm/test/storm/starter/tools/MockTupleHelpers.java @@ -0,0 +1,23 @@ +package storm.starter.tools; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import backtype.storm.Constants; +import backtype.storm.tuple.Tuple; + +public final class MockTupleHelpers { + + private MockTupleHelpers() { + } + + public static Tuple mockTickTuple() { + return mockTuple(Constants.SYSTEM_COMPONENT_ID, Constants.SYSTEM_TICK_STREAM_ID); + } + + public static Tuple mockTuple(String componentId, String streamId) { + Tuple tuple = mock(Tuple.class); + when(tuple.getSourceComponent()).thenReturn(componentId); + when(tuple.getSourceStreamId()).thenReturn(streamId); + return tuple; + } +} diff --git a/src/jvm/test/storm/starter/tools/NthLastModifiedTimeTrackerTest.java b/src/jvm/test/storm/starter/tools/NthLastModifiedTimeTrackerTest.java new file mode 100644 index 0000000000..77f8f717d2 --- /dev/null +++ b/src/jvm/test/storm/starter/tools/NthLastModifiedTimeTrackerTest.java @@ -0,0 +1,112 @@ +package storm.starter.tools; + +import static org.fest.assertions.api.Assertions.assertThat; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import backtype.storm.utils.Time; + +public class NthLastModifiedTimeTrackerTest { + + private static final int ANY_NUM_TIMES_TO_TRACK = 3; + private static final int MILLIS_IN_SEC = 1000; + + @DataProvider + public Object[][] illegalNumTimesData() { + return new Object[][] { { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalNumTimesData") + public void negativeOrZeroNumTimesToTrackShouldThrowIAE(int numTimesToTrack) { + new NthLastModifiedTimeTracker(numTimesToTrack); + } + + @DataProvider + public Object[][] legalNumTimesData() { + return new Object[][] { { 1 }, { 2 }, { 3 }, { 20 } }; + } + + @Test(dataProvider = "legalNumTimesData") + public void positiveNumTimesToTrackShouldBeOk(int numTimesToTrack) { + new NthLastModifiedTimeTracker(numTimesToTrack); + } + + @DataProvider + public Object[][] whenNotYetMarkedAsModifiedData() { + return new Object[][] { { 0 }, { 1 }, { 2 }, { 3 }, { 4 }, { 5 }, { 8 }, { 10 } }; + } + + @Test(dataProvider = "whenNotYetMarkedAsModifiedData") + public void shouldReturnCorrectModifiedTimeEvenWhenNotYetMarkedAsModified(int secondsToAdvance) { + // given + Time.startSimulating(); + NthLastModifiedTimeTracker tracker = new NthLastModifiedTimeTracker(ANY_NUM_TIMES_TO_TRACK); + + // when + advanceSimulatedTimeBy(secondsToAdvance); + int seconds = tracker.secondsSinceOldestModification(); + + // then + assertThat(seconds).isEqualTo(secondsToAdvance); + + // cleanup + Time.stopSimulating(); + } + + @DataProvider + public Object[][] simulatedTrackerIterations() { + return new Object[][] { + { 1, new int[] { 0, 1 }, new int[] { 0, 0 } }, + { 1, new int[] { 0, 2 }, new int[] { 0, 0 } }, + { 2, new int[] { 2, 2 }, new int[] { 2, 2 } }, + { 2, new int[] { 0, 4 }, new int[] { 0, 4 } }, + { 1, new int[] { 1, 1, 1, 1, 1, 1, 1 }, new int[] { 0, 0, 0, 0, 0, 0, 0 } }, + { 1, new int[] { 1, 2, 3, 4, 5, 6, 7 }, new int[] { 0, 0, 0, 0, 0, 0, 0 } }, + { 2, new int[] { 1, 1, 1, 1, 1, 1, 1 }, new int[] { 1, 1, 1, 1, 1, 1, 1 } }, + { 2, new int[] { 2, 2, 2, 2, 2, 2, 2 }, new int[] { 2, 2, 2, 2, 2, 2, 2 } }, + { 2, new int[] { 1, 2, 3, 4, 5, 6, 7 }, new int[] { 1, 2, 3, 4, 5, 6, 7 } }, + { 3, new int[] { 1, 1, 1, 1, 1, 1, 1 }, new int[] { 1, 2, 2, 2, 2, 2, 2 } }, + { 3, new int[] { 1, 2, 3, 4, 5, 6, 7 }, new int[] { 1, 3, 5, 7, 9, 11, 13 } }, + { 3, new int[] { 2, 2, 2, 2, 2, 2, 2 }, new int[] { 2, 4, 4, 4, 4, 4, 4 } }, + { 4, new int[] { 1, 1, 1, 1, 1, 1, 1 }, new int[] { 1, 2, 3, 3, 3, 3, 3 } }, + { 4, new int[] { 1, 2, 3, 4, 5, 6, 7 }, new int[] { 1, 3, 6, 9, 12, 15, 18 } }, + { 4, new int[] { 2, 2, 2, 2, 2, 2, 2 }, new int[] { 2, 4, 6, 6, 6, 6, 6 } }, + { 5, new int[] { 1, 1, 1, 1, 1, 1, 1 }, new int[] { 1, 2, 3, 4, 4, 4, 4 } }, + { 5, new int[] { 1, 2, 3, 4, 5, 6, 7 }, new int[] { 1, 3, 6, 10, 14, 18, 22 } }, + { 5, new int[] { 2, 2, 2, 2, 2, 2, 2 }, new int[] { 2, 4, 6, 8, 8, 8, 8 } }, + { 6, new int[] { 1, 1, 1, 1, 1, 1, 1 }, new int[] { 1, 2, 3, 4, 5, 5, 5 } }, + { 6, new int[] { 1, 2, 3, 4, 5, 6, 7 }, new int[] { 1, 3, 6, 10, 15, 20, 25 } }, + { 6, new int[] { 2, 2, 2, 2, 2, 2, 2 }, new int[] { 2, 4, 6, 8, 10, 10, 10 } }, + { 3, new int[] { 1, 2, 3 }, new int[] { 1, 3, 5 } } }; + } + + @Test(dataProvider = "simulatedTrackerIterations") + public void shouldReturnCorrectModifiedTimeWhenMarkedAsModified(int numTimesToTrack, + int[] secondsToAdvancePerIteration, int[] expLastModifiedTimes) { + // given + Time.startSimulating(); + NthLastModifiedTimeTracker tracker = new NthLastModifiedTimeTracker(numTimesToTrack); + + int[] modifiedTimes = new int[expLastModifiedTimes.length]; + + // when + int i = 0; + for (int secondsToAdvance : secondsToAdvancePerIteration) { + advanceSimulatedTimeBy(secondsToAdvance); + tracker.markAsModified(); + modifiedTimes[i] = tracker.secondsSinceOldestModification(); + i++; + } + + // then + assertThat(modifiedTimes).isEqualTo(expLastModifiedTimes); + + // cleanup + Time.stopSimulating(); + } + + private void advanceSimulatedTimeBy(int seconds) { + Time.advanceTime(seconds * MILLIS_IN_SEC); + } +} diff --git a/src/jvm/test/storm/starter/tools/RankableObjectWithFieldsTest.java b/src/jvm/test/storm/starter/tools/RankableObjectWithFieldsTest.java new file mode 100644 index 0000000000..b2360c13f1 --- /dev/null +++ b/src/jvm/test/storm/starter/tools/RankableObjectWithFieldsTest.java @@ -0,0 +1,225 @@ +package storm.starter.tools; + +import static org.fest.assertions.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import backtype.storm.tuple.Tuple; + +import com.google.common.collect.Lists; + +public class RankableObjectWithFieldsTest { + + private static final Object ANY_OBJECT = new Object(); + private static final long ANY_COUNT = 271; + private static final String ANY_FIELD = "someAdditionalField"; + private static final int GREATER_THAN = 1; + private static final int EQUAL_TO = 0; + private static final int SMALLER_THAN = -1; + + @Test(expectedExceptions = IllegalArgumentException.class) + public void constructorWithNullObjectAndNoFieldsShouldThrowIAE() { + new RankableObjectWithFields(null, ANY_COUNT); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void constructorWithNullObjectAndFieldsShouldThrowIAE() { + Object someAdditionalField = new Object(); + new RankableObjectWithFields(null, ANY_COUNT, someAdditionalField); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void constructorWithNegativeCountAndNoFieldsShouldThrowIAE() { + new RankableObjectWithFields(ANY_OBJECT, -1); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void constructorWithNegativeCountAndFieldsShouldThrowIAE() { + Object someAdditionalField = new Object(); + new RankableObjectWithFields(ANY_OBJECT, -1, someAdditionalField); + } + + @Test + public void shouldBeEqualToItself() { + RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT); + assertThat(r).isEqualTo(r); + } + + @DataProvider + public Object[][] otherClassesData() { + return new Object[][] { + { new String("foo") }, + { new Object() }, + { Integer.valueOf(4) }, + { Lists.newArrayList(7, 8, 9) } }; + } + + @Test(dataProvider = "otherClassesData") + public void shouldNotBeEqualToInstancesOfOtherClasses(Object notARankable) { + RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT); + assertFalse(r.equals(notARankable), r + " is equal to " + notARankable + " but it should not be"); + } + + @DataProvider + public Object[][] falseDuplicatesData() { + return new Object[][] { + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1) }, + { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("Foo", 1) }, + { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("FOO", 1) }, + { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 1) }, + { new RankableObjectWithFields("", 0), new RankableObjectWithFields("", 1) }, + { new RankableObjectWithFields("", 1), new RankableObjectWithFields("bar", 1) } }; + } + + @Test(dataProvider = "falseDuplicatesData") + public void shouldNotBeEqualToFalseDuplicates(RankableObjectWithFields r, RankableObjectWithFields falseDuplicate) { + assertFalse(r.equals(falseDuplicate), r + " is equal to " + falseDuplicate + " but it should not be"); + } + + @Test(dataProvider = "falseDuplicatesData") + public void shouldHaveDifferentHashCodeThanFalseDuplicates(RankableObjectWithFields r, + RankableObjectWithFields falseDuplicate) { + assertThat(r.hashCode()).isNotEqualTo(falseDuplicate.hashCode()); + } + + @DataProvider + public Object[][] trueDuplicatesData() { + return new Object[][] { + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0) }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0, "someOtherField") }, + { + new RankableObjectWithFields("foo", 0, "someField"), + new RankableObjectWithFields("foo", 0, "someOtherField") } }; + } + + @Test(dataProvider = "trueDuplicatesData") + public void shouldBeEqualToTrueDuplicates(RankableObjectWithFields r, RankableObjectWithFields trueDuplicate) { + assertTrue(r.equals(trueDuplicate), r + " is not equal to " + trueDuplicate + " but it should be"); + } + + @Test(dataProvider = "trueDuplicatesData") + public void shouldHaveSameHashCodeAsTrueDuplicates(RankableObjectWithFields r, + RankableObjectWithFields trueDuplicate) { + assertThat(r.hashCode()).isEqualTo(trueDuplicate.hashCode()); + } + + @DataProvider + public Object[][] compareToData() { + return new Object[][] { + { new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("foo", 0), GREATER_THAN }, + { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("foo", 0), GREATER_THAN }, + { new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("bar", 0), GREATER_THAN }, + { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 0), GREATER_THAN }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0), EQUAL_TO }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 0), EQUAL_TO }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1000), SMALLER_THAN }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1), SMALLER_THAN }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 1), SMALLER_THAN }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 1000), SMALLER_THAN }, }; + } + + @Test(dataProvider = "compareToData") + public void verifyCompareTo(RankableObjectWithFields first, RankableObjectWithFields second, int expCompareToValue) { + assertThat(first.compareTo(second)).isEqualTo(expCompareToValue); + } + + @DataProvider + public Object[][] toStringData() { + return new Object[][] { { new String("foo"), 0L }, { new String("BAR"), 8L } }; + } + + @Test(dataProvider = "toStringData") + public void toStringShouldContainStringRepresentationsOfObjectAndCount(Object obj, long count) { + // given + RankableObjectWithFields r = new RankableObjectWithFields(obj, count); + + // when + String strRepresentation = r.toString(); + + // then + assertThat(strRepresentation).contains(obj.toString()).contains("" + count); + } + + @Test + public void shouldReturnTheObject() { + // given + RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD); + + // when + Object obj = r.getObject(); + + // then + assertThat(obj).isEqualTo(ANY_OBJECT); + } + + @Test + public void shouldReturnTheCount() { + // given + RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD); + + // when + long count = r.getCount(); + + // then + assertThat(count).isEqualTo(ANY_COUNT); + } + + @DataProvider + public Object[][] fieldsData() { + return new Object[][] { + { ANY_OBJECT, ANY_COUNT, new Object[] { ANY_FIELD } }, + { "quux", 42L, new Object[] { "one", "two", "three" } } }; + } + + @Test(dataProvider = "fieldsData") + public void shouldReturnTheFields(Object obj, long count, Object[] fields) { + // given + RankableObjectWithFields r = new RankableObjectWithFields(obj, count, fields); + + // when + List actualFields = r.getFields(); + + // then + assertThat(actualFields).isEqualTo(Lists.newArrayList(fields)); + } + + @Test(expectedExceptions = UnsupportedOperationException.class) + public void fieldsShouldBeImmutable() { + // given + RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD); + + // when + List fields = r.getFields(); + // try to modify the list, which should fail + fields.remove(0); + + // then (exception) + } + + @Test + public void shouldCreateRankableObjectFromTuple() { + // given + Tuple tuple = mock(Tuple.class); + List tupleValues = Lists.newArrayList(ANY_OBJECT, ANY_COUNT, ANY_FIELD); + when(tuple.getValues()).thenReturn(tupleValues); + + // when + RankableObjectWithFields r = RankableObjectWithFields.from(tuple); + + // then + assertThat(r.getObject()).isEqualTo(ANY_OBJECT); + assertThat(r.getCount()).isEqualTo(ANY_COUNT); + List fields = new ArrayList(); + fields.add(ANY_FIELD); + assertThat(r.getFields()).isEqualTo(fields); + + } +} diff --git a/src/jvm/test/storm/starter/tools/RankingsTest.java b/src/jvm/test/storm/starter/tools/RankingsTest.java new file mode 100644 index 0000000000..682b504482 --- /dev/null +++ b/src/jvm/test/storm/starter/tools/RankingsTest.java @@ -0,0 +1,206 @@ +package storm.starter.tools; + +import static org.fest.assertions.api.Assertions.assertThat; + +import java.util.List; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import com.google.common.collect.Lists; + +public class RankingsTest { + + private static final int ANY_TOPN = 42; + private static final Rankable ANY_RANKABLE = new RankableObjectWithFields("someObject", ANY_TOPN); + private static final Rankable A = new RankableObjectWithFields("A", 1); + private static final Rankable B = new RankableObjectWithFields("B", 2); + private static final Rankable C = new RankableObjectWithFields("C", 3); + private static final Rankable D = new RankableObjectWithFields("D", 4); + private static final Rankable E = new RankableObjectWithFields("E", 5); + private static final Rankable F = new RankableObjectWithFields("F", 6); + private static final Rankable G = new RankableObjectWithFields("G", 7); + private static final Rankable H = new RankableObjectWithFields("H", 8); + + @DataProvider + public Object[][] illegalTopNData() { + return new Object[][] { { 0 }, { -1 }, { -2 }, { -10 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopNData") + public void constructorWithNegativeOrZeroTopNShouldThrowIAE(int topN) { + new Rankings(topN); + } + + @DataProvider + public Object[][] legalTopNData() { + return new Object[][] { { 1 }, { 2 }, { 1000 }, { 1000000 } }; + } + + @Test(dataProvider = "legalTopNData") + public void constructorWithPositiveTopNShouldBeOk(int topN) { + // given/when + Rankings rankings = new Rankings(topN); + + // then + assertThat(rankings.maxSize()).isEqualTo(topN); + } + + @Test + public void shouldHaveDefaultConstructor() { + new Rankings(); + } + + @Test + public void defaultConstructorShouldSetPositiveTopN() { + // given/when + Rankings rankings = new Rankings(); + + // then + assertThat(rankings.maxSize()).isGreaterThan(0); + } + + @DataProvider + public Object[][] rankingsGrowData() { + return new Object[][] { + { + 2, + Lists.newArrayList(new RankableObjectWithFields("A", 1), new RankableObjectWithFields("B", 2), + new RankableObjectWithFields("C", 3)) }, + { + 2, + Lists.newArrayList(new RankableObjectWithFields("A", 1), new RankableObjectWithFields("B", 2), + new RankableObjectWithFields("C", 3), new RankableObjectWithFields("D", 4)) } }; + } + + @Test(dataProvider = "rankingsGrowData") + public void sizeOfRankingsShouldNotGrowBeyondTopN(int topN, List rankables) { + // sanity check of the provided test data + assertThat(rankables.size()).overridingErrorMessage( + "The supplied test data is not correct: the number of rankables <%d> should be greater than <%d>", + rankables.size(), topN).isGreaterThan(topN); + + // given + Rankings rankings = new Rankings(topN); + + // when + for (Rankable r : rankables) { + rankings.updateWith(r); + } + + // then + assertThat(rankings.size()).isLessThanOrEqualTo(rankings.maxSize()); + } + + @DataProvider + public Object[][] simulatedRankingsData() { + return new Object[][] { + { Lists.newArrayList(A), Lists.newArrayList(A) }, + { Lists.newArrayList(B, D, A, C), Lists.newArrayList(D, C, B, A) }, + { Lists.newArrayList(B, F, A, C, D, E), Lists.newArrayList(F, E, D, C, B, A) }, + { Lists.newArrayList(G, B, F, A, C, D, E, H), Lists.newArrayList(H, G, F, E, D, C, B, A) } }; + } + + @Test(dataProvider = "simulatedRankingsData") + public void shouldCorrectlyRankWhenUpdatedWithRankables(List unsorted, List expSorted) { + // given + Rankings rankings = new Rankings(unsorted.size()); + + // when + for (Rankable r : unsorted) { + rankings.updateWith(r); + } + + // then + assertThat(rankings.getRankings()).isEqualTo(expSorted); + } + + @Test(dataProvider = "simulatedRankingsData") + public void shouldCorrectlyRankWhenEmptyAndUpdatedWithOtherRankings(List unsorted, + List expSorted) { + // given + Rankings rankings = new Rankings(unsorted.size()); + Rankings otherRankings = new Rankings(rankings.maxSize()); + for (Rankable r : unsorted) { + otherRankings.updateWith(r); + } + + // when + rankings.updateWith(otherRankings); + + // then + assertThat(rankings.getRankings()).isEqualTo(expSorted); + } + + @Test(dataProvider = "simulatedRankingsData") + public void shouldCorrectlyRankWhenUpdatedWithEmptyOtherRankings(List unsorted, List expSorted) { + // given + Rankings rankings = new Rankings(unsorted.size()); + for (Rankable r : unsorted) { + rankings.updateWith(r); + } + Rankings emptyRankings = new Rankings(ANY_TOPN); + + // when + rankings.updateWith(emptyRankings); + + // then + assertThat(rankings.getRankings()).isEqualTo(expSorted); + } + + @DataProvider + public Object[][] simulatedRankingsAndOtherRankingsData() { + return new Object[][] { + { Lists.newArrayList(A), Lists.newArrayList(A), Lists.newArrayList(A) }, + { Lists.newArrayList(A, C), Lists.newArrayList(B, D), Lists.newArrayList(D, C, B, A) }, + { Lists.newArrayList(B, F, A), Lists.newArrayList(C, D, E), Lists.newArrayList(F, E, D, C, B, A) }, + { + Lists.newArrayList(G, B, F, A, C), + Lists.newArrayList(D, E, H), + Lists.newArrayList(H, G, F, E, D, C, B, A) } }; + } + + @Test(dataProvider = "simulatedRankingsAndOtherRankingsData") + public void shouldCorrectlyRankWhenNotEmptyAndUpdatedWithOtherRankings(List unsorted, + List unsortedForOtherRankings, List expSorted) { + // given + Rankings rankings = new Rankings(expSorted.size()); + for (Rankable r : unsorted) { + rankings.updateWith(r); + } + Rankings otherRankings = new Rankings(unsortedForOtherRankings.size()); + for (Rankable r : unsortedForOtherRankings) { + otherRankings.updateWith(r); + } + + // when + rankings.updateWith(otherRankings); + + // then + assertThat(rankings.getRankings()).isEqualTo(expSorted); + } + + @DataProvider + public Object[][] duplicatesData() { + Rankable A1 = new RankableObjectWithFields("A", 1); + Rankable A2 = new RankableObjectWithFields("A", 2); + Rankable A3 = new RankableObjectWithFields("A", 3); + return new Object[][] { + { Lists.newArrayList(ANY_RANKABLE, ANY_RANKABLE, ANY_RANKABLE) }, + { Lists.newArrayList(A1, A2, A3) }, }; + } + + @Test(dataProvider = "duplicatesData") + public void shouldNotRankDuplicateObjectsMoreThanOnce(List duplicates) { + // given + Rankings rankings = new Rankings(duplicates.size()); + + // when + for (Rankable r : duplicates) { + rankings.updateWith(r); + } + + // then + assertThat(rankings.size()).isEqualTo(1); + } +} diff --git a/src/jvm/test/storm/starter/tools/SlidingWindowCounterTest.java b/src/jvm/test/storm/starter/tools/SlidingWindowCounterTest.java new file mode 100644 index 0000000000..fddff591b7 --- /dev/null +++ b/src/jvm/test/storm/starter/tools/SlidingWindowCounterTest.java @@ -0,0 +1,88 @@ +package storm.starter.tools; + +import static org.fest.assertions.api.Assertions.assertThat; + +import java.util.Map; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class SlidingWindowCounterTest { + + private static final int ANY_WINDOW_LENGTH_IN_SLOTS = 2; + private static final Object ANY_OBJECT = "ANY_OBJECT"; + + @DataProvider + public Object[][] illegalWindowLengths() { + return new Object[][] { { -10 }, { -3 }, { -2 }, { -1 }, { 0 }, { 1 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalWindowLengths") + public void lessThanTwoSlotsShouldThrowIAE(int windowLengthInSlots) { + new SlidingWindowCounter(windowLengthInSlots); + } + + @DataProvider + public Object[][] legalWindowLengths() { + return new Object[][] { { 2 }, { 3 }, { 20 } }; + } + + @Test(dataProvider = "legalWindowLengths") + public void twoOrMoreSlotsShouldBeValid(int windowLengthInSlots) { + new SlidingWindowCounter(windowLengthInSlots); + } + + public void newInstanceShouldHaveEmptyCounts() { + // given + SlidingWindowCounter counter = new SlidingWindowCounter(ANY_WINDOW_LENGTH_IN_SLOTS); + + // when + Map counts = counter.getCountsThenAdvanceWindow(); + + // then + assertThat(counts).isEmpty(); + } + + @DataProvider + public Object[][] simulatedCounterIterations() { + return new Object[][] { + { 2, new int[] { 3, 2, 0, 0, 1, 0, 0, 0 }, new long[] { 3, 5, 2, 0, 1, 1, 0, 0 } }, + { 3, new int[] { 3, 2, 0, 0, 1, 0, 0, 0 }, new long[] { 3, 5, 5, 2, 1, 1, 1, 0 } }, + { 4, new int[] { 3, 2, 0, 0, 1, 0, 0, 0 }, new long[] { 3, 5, 5, 5, 3, 1, 1, 1 } }, + { 5, new int[] { 3, 2, 0, 0, 1, 0, 0, 0 }, new long[] { 3, 5, 5, 5, 6, 3, 1, 1 } }, + { + 5, + new int[] { 3, 11, 5, 13, 7, 17, 0, 3, 50, 600, 7000 }, + new long[] { 3, 14, 19, 32, 39, 53, 42, 40, 77, 670, 7653 } }, }; + } + + @Test(dataProvider = "simulatedCounterIterations") + public void testCounterWithSimulatedRuns(int windowLengthInSlots, int[] incrementsPerIteration, + long[] expCountsPerIteration) { + // given + SlidingWindowCounter counter = new SlidingWindowCounter(windowLengthInSlots); + int numIterations = incrementsPerIteration.length; + + for (int i = 0; i < numIterations; i++) { + int numIncrements = incrementsPerIteration[i]; + long expCounts = expCountsPerIteration[i]; + + // given (for this iteration) + for (int j = 0; j < numIncrements; j++) { + counter.incrementCount(ANY_OBJECT); + } + + // when (for this iteration) + Map counts = counter.getCountsThenAdvanceWindow(); + + // then (for this iteration) + if (expCounts == 0) { + assertThat(counts).doesNotContainKey(ANY_OBJECT); + } + else { + assertThat(counts.get(ANY_OBJECT)).isEqualTo(expCounts); + } + } + } + +} diff --git a/src/jvm/test/storm/starter/tools/SlotBasedCounterTest.java b/src/jvm/test/storm/starter/tools/SlotBasedCounterTest.java new file mode 100644 index 0000000000..c3e51c1841 --- /dev/null +++ b/src/jvm/test/storm/starter/tools/SlotBasedCounterTest.java @@ -0,0 +1,163 @@ +package storm.starter.tools; + +import static org.fest.assertions.api.Assertions.assertThat; + +import java.util.Map; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class SlotBasedCounterTest { + + private static final int ANY_NUM_SLOTS = 1; + private static final int ANY_SLOT = 0; + private static final Object ANY_OBJECT = "ANY_OBJECT"; + + @DataProvider + public Object[][] illegalNumSlotsData() { + return new Object[][] { { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalNumSlotsData") + public void negativeOrZeroNumSlotsShouldThrowIAE(int numSlots) { + new SlotBasedCounter(numSlots); + } + + @DataProvider + public Object[][] legalNumSlotsData() { + return new Object[][] { { 1 }, { 2 }, { 3 }, { 20 } }; + } + + @Test(dataProvider = "legalNumSlotsData") + public void positiveNumSlotsShouldBeOk(int numSlots) { + new SlotBasedCounter(numSlots); + } + + @Test + public void newInstanceShouldHaveEmptyCounts() { + // given + SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); + + // when + Map counts = counter.getCounts(); + + // then + assertThat(counts).isEmpty(); + } + + @Test + public void shouldReturnNonEmptyCountsWhenAtLeastOneObjectWasCounted() { + // given + SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); + counter.incrementCount(ANY_OBJECT, ANY_SLOT); + + // when + Map counts = counter.getCounts(); + + // then + assertThat(counts).isNotEmpty(); + + // additional tests that go beyond what this test is primarily about + assertThat(counts.size()).isEqualTo(1); + assertThat(counts.get(ANY_OBJECT)).isEqualTo(1); + } + + @DataProvider + public Object[][] incrementCountData() { + return new Object[][] { { new String[] { "foo", "bar" }, new int[] { 3, 2 } } }; + } + + @Test(dataProvider = "incrementCountData") + public void shouldIncrementCount(Object[] objects, int[] expCounts) { + // given + SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); + + // when + for (int i = 0; i < objects.length; i++) { + Object obj = objects[i]; + int numIncrements = expCounts[i]; + for (int j = 0; j < numIncrements; j++) { + counter.incrementCount(obj, ANY_SLOT); + } + } + + // then + for (int i = 0; i < objects.length; i++) { + assertThat(counter.getCount(objects[i], ANY_SLOT)).isEqualTo(expCounts[i]); + } + assertThat(counter.getCount("nonexistentObject", ANY_SLOT)).isEqualTo(0); + } + + @Test + public void shouldReturnZeroForNonexistentObject() { + // given + SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); + + // when + counter.incrementCount("somethingElse", ANY_SLOT); + + // then + assertThat(counter.getCount("nonexistentObject", ANY_SLOT)).isEqualTo(0); + } + + @Test + public void shouldIncrementCountOnlyOneSlotAtATime() { + // given + int numSlots = 3; + Object obj = Long.valueOf(10); + SlotBasedCounter counter = new SlotBasedCounter(numSlots); + + // when (empty) + // then + assertThat(counter.getCount(obj, 0)).isEqualTo(0); + assertThat(counter.getCount(obj, 1)).isEqualTo(0); + assertThat(counter.getCount(obj, 2)).isEqualTo(0); + + // when + counter.incrementCount(obj, 1); + + // then + assertThat(counter.getCount(obj, 0)).isEqualTo(0); + assertThat(counter.getCount(obj, 1)).isEqualTo(1); + assertThat(counter.getCount(obj, 2)).isEqualTo(0); + } + + @Test + public void wipeSlotShouldSetAllCountsInSlotToZero() { + // given + SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); + Object countWasOne = "countWasOne"; + Object countWasThree = "countWasThree"; + counter.incrementCount(countWasOne, ANY_SLOT); + counter.incrementCount(countWasThree, ANY_SLOT); + counter.incrementCount(countWasThree, ANY_SLOT); + counter.incrementCount(countWasThree, ANY_SLOT); + + // when + counter.wipeSlot(ANY_SLOT); + + // then + assertThat(counter.getCount(countWasOne, ANY_SLOT)).isEqualTo(0); + assertThat(counter.getCount(countWasThree, ANY_SLOT)).isEqualTo(0); + } + + @Test + public void wipeSlotShouldRemoveAnyObjectsWithZeroTotalCount() { + // given + SlotBasedCounter counter = new SlotBasedCounter(2); + int wipeSlot = 0; + int otherSlot = 1; + Object willBeRemoved = "willBeRemoved"; + Object willContinueToBeTracked = "willContinueToBeTracked"; + counter.incrementCount(willBeRemoved, wipeSlot); + counter.incrementCount(willContinueToBeTracked, wipeSlot); + counter.incrementCount(willContinueToBeTracked, otherSlot); + + // when + counter.wipeSlot(wipeSlot); + + // then + assertThat(counter.getCounts()).doesNotContainKey(willBeRemoved); + assertThat(counter.getCounts()).containsKey(willContinueToBeTracked); + } +} From 09081b3d2b1da3dcdd3b97b796b8adf56544583a Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Tue, 11 Dec 2012 10:06:22 +0100 Subject: [PATCH 151/349] Move code back to src/jvm and place unit tests into test/jvm --- m2-pom.xml | 4 ++-- src/jvm/{main => }/storm/starter/BasicDRPCTopology.java | 0 src/jvm/{main => }/storm/starter/ExclamationTopology.java | 0 src/jvm/{main => }/storm/starter/ManualDRPC.java | 0 src/jvm/{main => }/storm/starter/PrintSampleStream.java | 0 src/jvm/{main => }/storm/starter/ReachTopology.java | 0 src/jvm/{main => }/storm/starter/RollingTopWords.java | 0 src/jvm/{main => }/storm/starter/SingleJoinExample.java | 0 .../{main => }/storm/starter/TransactionalGlobalCount.java | 0 src/jvm/{main => }/storm/starter/TransactionalWords.java | 0 src/jvm/{main => }/storm/starter/WordCountTopology.java | 0 src/jvm/{main => }/storm/starter/bolt/AbstractRankerBolt.java | 0 .../storm/starter/bolt/IntermediateRankingsBolt.java | 0 src/jvm/{main => }/storm/starter/bolt/PrinterBolt.java | 0 src/jvm/{main => }/storm/starter/bolt/RollingCountBolt.java | 0 src/jvm/{main => }/storm/starter/bolt/SingleJoinBolt.java | 0 src/jvm/{main => }/storm/starter/bolt/TotalRankingsBolt.java | 0 .../{main => }/storm/starter/spout/RandomSentenceSpout.java | 0 .../{main => }/storm/starter/spout/TwitterSampleSpout.java | 0 .../storm/starter/tools/NthLastModifiedTimeTracker.java | 0 src/jvm/{main => }/storm/starter/tools/Rankable.java | 0 .../storm/starter/tools/RankableObjectWithFields.java | 0 src/jvm/{main => }/storm/starter/tools/Rankings.java | 0 .../{main => }/storm/starter/tools/SlidingWindowCounter.java | 0 src/jvm/{main => }/storm/starter/tools/SlotBasedCounter.java | 0 src/jvm/{main => }/storm/starter/trident/TridentReach.java | 0 .../{main => }/storm/starter/trident/TridentWordCount.java | 0 src/jvm/{main => }/storm/starter/util/StormRunner.java | 0 src/jvm/{main => }/storm/starter/util/TupleHelpers.java | 0 .../jvm}/storm/starter/bolt/IntermediateRankingsBoltTest.java | 0 .../jvm}/storm/starter/bolt/RollingCountBoltTest.java | 0 .../jvm}/storm/starter/bolt/TotalRankingsBoltTest.java | 0 .../jvm}/storm/starter/tools/MockTupleHelpers.java | 0 .../storm/starter/tools/NthLastModifiedTimeTrackerTest.java | 0 .../storm/starter/tools/RankableObjectWithFieldsTest.java | 0 .../test => test/jvm}/storm/starter/tools/RankingsTest.java | 0 .../jvm}/storm/starter/tools/SlidingWindowCounterTest.java | 0 .../jvm}/storm/starter/tools/SlotBasedCounterTest.java | 0 38 files changed, 2 insertions(+), 2 deletions(-) rename src/jvm/{main => }/storm/starter/BasicDRPCTopology.java (100%) rename src/jvm/{main => }/storm/starter/ExclamationTopology.java (100%) rename src/jvm/{main => }/storm/starter/ManualDRPC.java (100%) rename src/jvm/{main => }/storm/starter/PrintSampleStream.java (100%) rename src/jvm/{main => }/storm/starter/ReachTopology.java (100%) rename src/jvm/{main => }/storm/starter/RollingTopWords.java (100%) rename src/jvm/{main => }/storm/starter/SingleJoinExample.java (100%) rename src/jvm/{main => }/storm/starter/TransactionalGlobalCount.java (100%) rename src/jvm/{main => }/storm/starter/TransactionalWords.java (100%) rename src/jvm/{main => }/storm/starter/WordCountTopology.java (100%) rename src/jvm/{main => }/storm/starter/bolt/AbstractRankerBolt.java (100%) rename src/jvm/{main => }/storm/starter/bolt/IntermediateRankingsBolt.java (100%) rename src/jvm/{main => }/storm/starter/bolt/PrinterBolt.java (100%) rename src/jvm/{main => }/storm/starter/bolt/RollingCountBolt.java (100%) rename src/jvm/{main => }/storm/starter/bolt/SingleJoinBolt.java (100%) rename src/jvm/{main => }/storm/starter/bolt/TotalRankingsBolt.java (100%) rename src/jvm/{main => }/storm/starter/spout/RandomSentenceSpout.java (100%) rename src/jvm/{main => }/storm/starter/spout/TwitterSampleSpout.java (100%) rename src/jvm/{main => }/storm/starter/tools/NthLastModifiedTimeTracker.java (100%) rename src/jvm/{main => }/storm/starter/tools/Rankable.java (100%) rename src/jvm/{main => }/storm/starter/tools/RankableObjectWithFields.java (100%) rename src/jvm/{main => }/storm/starter/tools/Rankings.java (100%) rename src/jvm/{main => }/storm/starter/tools/SlidingWindowCounter.java (100%) rename src/jvm/{main => }/storm/starter/tools/SlotBasedCounter.java (100%) rename src/jvm/{main => }/storm/starter/trident/TridentReach.java (100%) rename src/jvm/{main => }/storm/starter/trident/TridentWordCount.java (100%) rename src/jvm/{main => }/storm/starter/util/StormRunner.java (100%) rename src/jvm/{main => }/storm/starter/util/TupleHelpers.java (100%) rename {src/jvm/test => test/jvm}/storm/starter/bolt/IntermediateRankingsBoltTest.java (100%) rename {src/jvm/test => test/jvm}/storm/starter/bolt/RollingCountBoltTest.java (100%) rename {src/jvm/test => test/jvm}/storm/starter/bolt/TotalRankingsBoltTest.java (100%) rename {src/jvm/test => test/jvm}/storm/starter/tools/MockTupleHelpers.java (100%) rename {src/jvm/test => test/jvm}/storm/starter/tools/NthLastModifiedTimeTrackerTest.java (100%) rename {src/jvm/test => test/jvm}/storm/starter/tools/RankableObjectWithFieldsTest.java (100%) rename {src/jvm/test => test/jvm}/storm/starter/tools/RankingsTest.java (100%) rename {src/jvm/test => test/jvm}/storm/starter/tools/SlidingWindowCounterTest.java (100%) rename {src/jvm/test => test/jvm}/storm/starter/tools/SlotBasedCounterTest.java (100%) diff --git a/m2-pom.xml b/m2-pom.xml index 6afcbc77a4..984f171168 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -88,8 +88,8 @@ - src/jvm/main - src/jvm/test + src/jvm + test/jvm ${basedir}/multilang diff --git a/src/jvm/main/storm/starter/BasicDRPCTopology.java b/src/jvm/storm/starter/BasicDRPCTopology.java similarity index 100% rename from src/jvm/main/storm/starter/BasicDRPCTopology.java rename to src/jvm/storm/starter/BasicDRPCTopology.java diff --git a/src/jvm/main/storm/starter/ExclamationTopology.java b/src/jvm/storm/starter/ExclamationTopology.java similarity index 100% rename from src/jvm/main/storm/starter/ExclamationTopology.java rename to src/jvm/storm/starter/ExclamationTopology.java diff --git a/src/jvm/main/storm/starter/ManualDRPC.java b/src/jvm/storm/starter/ManualDRPC.java similarity index 100% rename from src/jvm/main/storm/starter/ManualDRPC.java rename to src/jvm/storm/starter/ManualDRPC.java diff --git a/src/jvm/main/storm/starter/PrintSampleStream.java b/src/jvm/storm/starter/PrintSampleStream.java similarity index 100% rename from src/jvm/main/storm/starter/PrintSampleStream.java rename to src/jvm/storm/starter/PrintSampleStream.java diff --git a/src/jvm/main/storm/starter/ReachTopology.java b/src/jvm/storm/starter/ReachTopology.java similarity index 100% rename from src/jvm/main/storm/starter/ReachTopology.java rename to src/jvm/storm/starter/ReachTopology.java diff --git a/src/jvm/main/storm/starter/RollingTopWords.java b/src/jvm/storm/starter/RollingTopWords.java similarity index 100% rename from src/jvm/main/storm/starter/RollingTopWords.java rename to src/jvm/storm/starter/RollingTopWords.java diff --git a/src/jvm/main/storm/starter/SingleJoinExample.java b/src/jvm/storm/starter/SingleJoinExample.java similarity index 100% rename from src/jvm/main/storm/starter/SingleJoinExample.java rename to src/jvm/storm/starter/SingleJoinExample.java diff --git a/src/jvm/main/storm/starter/TransactionalGlobalCount.java b/src/jvm/storm/starter/TransactionalGlobalCount.java similarity index 100% rename from src/jvm/main/storm/starter/TransactionalGlobalCount.java rename to src/jvm/storm/starter/TransactionalGlobalCount.java diff --git a/src/jvm/main/storm/starter/TransactionalWords.java b/src/jvm/storm/starter/TransactionalWords.java similarity index 100% rename from src/jvm/main/storm/starter/TransactionalWords.java rename to src/jvm/storm/starter/TransactionalWords.java diff --git a/src/jvm/main/storm/starter/WordCountTopology.java b/src/jvm/storm/starter/WordCountTopology.java similarity index 100% rename from src/jvm/main/storm/starter/WordCountTopology.java rename to src/jvm/storm/starter/WordCountTopology.java diff --git a/src/jvm/main/storm/starter/bolt/AbstractRankerBolt.java b/src/jvm/storm/starter/bolt/AbstractRankerBolt.java similarity index 100% rename from src/jvm/main/storm/starter/bolt/AbstractRankerBolt.java rename to src/jvm/storm/starter/bolt/AbstractRankerBolt.java diff --git a/src/jvm/main/storm/starter/bolt/IntermediateRankingsBolt.java b/src/jvm/storm/starter/bolt/IntermediateRankingsBolt.java similarity index 100% rename from src/jvm/main/storm/starter/bolt/IntermediateRankingsBolt.java rename to src/jvm/storm/starter/bolt/IntermediateRankingsBolt.java diff --git a/src/jvm/main/storm/starter/bolt/PrinterBolt.java b/src/jvm/storm/starter/bolt/PrinterBolt.java similarity index 100% rename from src/jvm/main/storm/starter/bolt/PrinterBolt.java rename to src/jvm/storm/starter/bolt/PrinterBolt.java diff --git a/src/jvm/main/storm/starter/bolt/RollingCountBolt.java b/src/jvm/storm/starter/bolt/RollingCountBolt.java similarity index 100% rename from src/jvm/main/storm/starter/bolt/RollingCountBolt.java rename to src/jvm/storm/starter/bolt/RollingCountBolt.java diff --git a/src/jvm/main/storm/starter/bolt/SingleJoinBolt.java b/src/jvm/storm/starter/bolt/SingleJoinBolt.java similarity index 100% rename from src/jvm/main/storm/starter/bolt/SingleJoinBolt.java rename to src/jvm/storm/starter/bolt/SingleJoinBolt.java diff --git a/src/jvm/main/storm/starter/bolt/TotalRankingsBolt.java b/src/jvm/storm/starter/bolt/TotalRankingsBolt.java similarity index 100% rename from src/jvm/main/storm/starter/bolt/TotalRankingsBolt.java rename to src/jvm/storm/starter/bolt/TotalRankingsBolt.java diff --git a/src/jvm/main/storm/starter/spout/RandomSentenceSpout.java b/src/jvm/storm/starter/spout/RandomSentenceSpout.java similarity index 100% rename from src/jvm/main/storm/starter/spout/RandomSentenceSpout.java rename to src/jvm/storm/starter/spout/RandomSentenceSpout.java diff --git a/src/jvm/main/storm/starter/spout/TwitterSampleSpout.java b/src/jvm/storm/starter/spout/TwitterSampleSpout.java similarity index 100% rename from src/jvm/main/storm/starter/spout/TwitterSampleSpout.java rename to src/jvm/storm/starter/spout/TwitterSampleSpout.java diff --git a/src/jvm/main/storm/starter/tools/NthLastModifiedTimeTracker.java b/src/jvm/storm/starter/tools/NthLastModifiedTimeTracker.java similarity index 100% rename from src/jvm/main/storm/starter/tools/NthLastModifiedTimeTracker.java rename to src/jvm/storm/starter/tools/NthLastModifiedTimeTracker.java diff --git a/src/jvm/main/storm/starter/tools/Rankable.java b/src/jvm/storm/starter/tools/Rankable.java similarity index 100% rename from src/jvm/main/storm/starter/tools/Rankable.java rename to src/jvm/storm/starter/tools/Rankable.java diff --git a/src/jvm/main/storm/starter/tools/RankableObjectWithFields.java b/src/jvm/storm/starter/tools/RankableObjectWithFields.java similarity index 100% rename from src/jvm/main/storm/starter/tools/RankableObjectWithFields.java rename to src/jvm/storm/starter/tools/RankableObjectWithFields.java diff --git a/src/jvm/main/storm/starter/tools/Rankings.java b/src/jvm/storm/starter/tools/Rankings.java similarity index 100% rename from src/jvm/main/storm/starter/tools/Rankings.java rename to src/jvm/storm/starter/tools/Rankings.java diff --git a/src/jvm/main/storm/starter/tools/SlidingWindowCounter.java b/src/jvm/storm/starter/tools/SlidingWindowCounter.java similarity index 100% rename from src/jvm/main/storm/starter/tools/SlidingWindowCounter.java rename to src/jvm/storm/starter/tools/SlidingWindowCounter.java diff --git a/src/jvm/main/storm/starter/tools/SlotBasedCounter.java b/src/jvm/storm/starter/tools/SlotBasedCounter.java similarity index 100% rename from src/jvm/main/storm/starter/tools/SlotBasedCounter.java rename to src/jvm/storm/starter/tools/SlotBasedCounter.java diff --git a/src/jvm/main/storm/starter/trident/TridentReach.java b/src/jvm/storm/starter/trident/TridentReach.java similarity index 100% rename from src/jvm/main/storm/starter/trident/TridentReach.java rename to src/jvm/storm/starter/trident/TridentReach.java diff --git a/src/jvm/main/storm/starter/trident/TridentWordCount.java b/src/jvm/storm/starter/trident/TridentWordCount.java similarity index 100% rename from src/jvm/main/storm/starter/trident/TridentWordCount.java rename to src/jvm/storm/starter/trident/TridentWordCount.java diff --git a/src/jvm/main/storm/starter/util/StormRunner.java b/src/jvm/storm/starter/util/StormRunner.java similarity index 100% rename from src/jvm/main/storm/starter/util/StormRunner.java rename to src/jvm/storm/starter/util/StormRunner.java diff --git a/src/jvm/main/storm/starter/util/TupleHelpers.java b/src/jvm/storm/starter/util/TupleHelpers.java similarity index 100% rename from src/jvm/main/storm/starter/util/TupleHelpers.java rename to src/jvm/storm/starter/util/TupleHelpers.java diff --git a/src/jvm/test/storm/starter/bolt/IntermediateRankingsBoltTest.java b/test/jvm/storm/starter/bolt/IntermediateRankingsBoltTest.java similarity index 100% rename from src/jvm/test/storm/starter/bolt/IntermediateRankingsBoltTest.java rename to test/jvm/storm/starter/bolt/IntermediateRankingsBoltTest.java diff --git a/src/jvm/test/storm/starter/bolt/RollingCountBoltTest.java b/test/jvm/storm/starter/bolt/RollingCountBoltTest.java similarity index 100% rename from src/jvm/test/storm/starter/bolt/RollingCountBoltTest.java rename to test/jvm/storm/starter/bolt/RollingCountBoltTest.java diff --git a/src/jvm/test/storm/starter/bolt/TotalRankingsBoltTest.java b/test/jvm/storm/starter/bolt/TotalRankingsBoltTest.java similarity index 100% rename from src/jvm/test/storm/starter/bolt/TotalRankingsBoltTest.java rename to test/jvm/storm/starter/bolt/TotalRankingsBoltTest.java diff --git a/src/jvm/test/storm/starter/tools/MockTupleHelpers.java b/test/jvm/storm/starter/tools/MockTupleHelpers.java similarity index 100% rename from src/jvm/test/storm/starter/tools/MockTupleHelpers.java rename to test/jvm/storm/starter/tools/MockTupleHelpers.java diff --git a/src/jvm/test/storm/starter/tools/NthLastModifiedTimeTrackerTest.java b/test/jvm/storm/starter/tools/NthLastModifiedTimeTrackerTest.java similarity index 100% rename from src/jvm/test/storm/starter/tools/NthLastModifiedTimeTrackerTest.java rename to test/jvm/storm/starter/tools/NthLastModifiedTimeTrackerTest.java diff --git a/src/jvm/test/storm/starter/tools/RankableObjectWithFieldsTest.java b/test/jvm/storm/starter/tools/RankableObjectWithFieldsTest.java similarity index 100% rename from src/jvm/test/storm/starter/tools/RankableObjectWithFieldsTest.java rename to test/jvm/storm/starter/tools/RankableObjectWithFieldsTest.java diff --git a/src/jvm/test/storm/starter/tools/RankingsTest.java b/test/jvm/storm/starter/tools/RankingsTest.java similarity index 100% rename from src/jvm/test/storm/starter/tools/RankingsTest.java rename to test/jvm/storm/starter/tools/RankingsTest.java diff --git a/src/jvm/test/storm/starter/tools/SlidingWindowCounterTest.java b/test/jvm/storm/starter/tools/SlidingWindowCounterTest.java similarity index 100% rename from src/jvm/test/storm/starter/tools/SlidingWindowCounterTest.java rename to test/jvm/storm/starter/tools/SlidingWindowCounterTest.java diff --git a/src/jvm/test/storm/starter/tools/SlotBasedCounterTest.java b/test/jvm/storm/starter/tools/SlotBasedCounterTest.java similarity index 100% rename from src/jvm/test/storm/starter/tools/SlotBasedCounterTest.java rename to test/jvm/storm/starter/tools/SlotBasedCounterTest.java From 6c326ba4ea855898e9211d1fafd0d66d1bbe33c1 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Tue, 11 Dec 2012 16:06:51 -0800 Subject: [PATCH 152/349] fix project.clj --- project.clj | 5 ++++- src/jvm/storm/starter/trident/TridentWordCount.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 1415ffb5cb..c6fb0fa84d 100644 --- a/project.clj +++ b/project.clj @@ -11,9 +11,12 @@ :dependencies [ ;; [org.twitter4j/twitter4j-core "2.2.6-SNAPSHOT"] ;; [org.twitter4j/twitter4j-stream "2.2.6-SNAPSHOT"] + [storm/trident-memcached "0.0.4-wip1"] + [commons-collections/commons-collections "3.2.1"] ] - :dev-dependencies [[storm "0.8.1"] + :dev-dependencies [[storm "0.8.2-wip18"] [org.clojure/clojure "1.4.0"] +; [com.thimbleware.jmemcached/jmemcached-cli "1.0.0"] ]) diff --git a/src/jvm/storm/starter/trident/TridentWordCount.java b/src/jvm/storm/starter/trident/TridentWordCount.java index 4712019ae7..268d41cb19 100644 --- a/src/jvm/storm/starter/trident/TridentWordCount.java +++ b/src/jvm/storm/starter/trident/TridentWordCount.java @@ -16,6 +16,7 @@ import storm.trident.operation.builtin.FilterNull; import storm.trident.operation.builtin.MapGet; import storm.trident.operation.builtin.Sum; +import storm.trident.planner.processor.StateQueryProcessor; import storm.trident.testing.MemoryMapState; import storm.trident.tuple.TridentTuple; @@ -57,7 +58,6 @@ public static StormTopology buildTopology(LocalDRPC drpc) { .each(new Fields("count"), new FilterNull()) .aggregate(new Fields("count"), new Sum(), new Fields("sum")) ; - return topology.build(); } From bde6ab22f9552515f4e225f9fa9b2a274d518dcd Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Tue, 11 Dec 2012 16:07:41 -0800 Subject: [PATCH 153/349] fix project.clj --- project.clj | 2 -- 1 file changed, 2 deletions(-) diff --git a/project.clj b/project.clj index c6fb0fa84d..746add3ea9 100644 --- a/project.clj +++ b/project.clj @@ -11,12 +11,10 @@ :dependencies [ ;; [org.twitter4j/twitter4j-core "2.2.6-SNAPSHOT"] ;; [org.twitter4j/twitter4j-stream "2.2.6-SNAPSHOT"] - [storm/trident-memcached "0.0.4-wip1"] [commons-collections/commons-collections "3.2.1"] ] :dev-dependencies [[storm "0.8.2-wip18"] [org.clojure/clojure "1.4.0"] -; [com.thimbleware.jmemcached/jmemcached-cli "1.0.0"] ]) From 2f90eff4e7fb7ce0b5738ef7860a1a3c601d4785 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Wed, 12 Dec 2012 17:07:46 +0100 Subject: [PATCH 154/349] SlidingWindowCounterTest: add missing @Test annotation to unit test --- test/jvm/storm/starter/tools/SlidingWindowCounterTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/jvm/storm/starter/tools/SlidingWindowCounterTest.java b/test/jvm/storm/starter/tools/SlidingWindowCounterTest.java index fddff591b7..2e8fb260f3 100644 --- a/test/jvm/storm/starter/tools/SlidingWindowCounterTest.java +++ b/test/jvm/storm/starter/tools/SlidingWindowCounterTest.java @@ -32,6 +32,7 @@ public void twoOrMoreSlotsShouldBeValid(int windowLengthInSlots) { new SlidingWindowCounter(windowLengthInSlots); } + @Test public void newInstanceShouldHaveEmptyCounts() { // given SlidingWindowCounter counter = new SlidingWindowCounter(ANY_WINDOW_LENGTH_IN_SLOTS); From 470ec47280a9fc1afc7e5d44ef047c7f6ed58713 Mon Sep 17 00:00:00 2001 From: Jason Jackson Date: Wed, 5 Dec 2012 01:53:45 -0800 Subject: [PATCH 155/349] multischeme related changes --- project.clj | 9 +++++---- src/jvm/storm/kafka/PartitionManager.java | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/project.clj b/project.clj index 72fa15400d..5b714a8b7c 100644 --- a/project.clj +++ b/project.clj @@ -1,13 +1,14 @@ -(defproject storm/storm-kafka "0.8.0-wip2-scala292" +(defproject storm/storm-kafka "0.9.0-wip5-scala292-multischeme" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} - :repositories {"releases" "http://artifactory.local.twitter.com/libs-releases-local" - "snapshots" "http://artifactory.local.twitter.com/libs-snapshots-local"} + :repositories {;;"releases" "http://artifactory.local.twitter.com/libs-releases-local" + ;;"snapshots" "http://artifactory.local.twitter.com/libs-snapshots-local" + "conjars" "http://conjars.org/repo/"} :dependencies [[com.twitter/kafka_2.9.2 "0.7.0" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.8.2-wip19"] + :dev-dependencies [[storm "0.9.0-wip9"] [org.slf4j/log4j-over-slf4j "1.6.6"] [ch.qos.logback/logback-classic "1.0.6"] [org.clojure/clojure "1.4.0"]] diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 4424b649d7..f0e50b5e51 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -84,7 +84,7 @@ public EmitState next(SpoutOutputCollector collector) { if(toEmit==null) { return EmitState.NO_EMITTED; } - Iterator> tups = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.msg.payload())); + Iterable> tups = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.msg.payload())); if(tups!=null) { for(List tup: tups) collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset)); From df380fb0186aa59e4d1f2f74e8f609c3bda23dc2 Mon Sep 17 00:00:00 2001 From: Jason Jackson Date: Mon, 17 Dec 2012 22:41:07 -0800 Subject: [PATCH 156/349] version bump 0.9.0-wip5 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 5b714a8b7c..0de3473bec 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.9.0-wip5-scala292-multischeme" +(defproject storm/storm-kafka "0.9.0-wip5-scala292" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} From 6e8c93d27bd18edbd7a0edc6188ad81921c28b57 Mon Sep 17 00:00:00 2001 From: Jason Jackson Date: Wed, 9 Jan 2013 00:55:43 -0800 Subject: [PATCH 157/349] removed twitter repositories from project clj --- project.clj | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/project.clj b/project.clj index 0de3473bec..6e269dd354 100644 --- a/project.clj +++ b/project.clj @@ -2,9 +2,7 @@ :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} - :repositories {;;"releases" "http://artifactory.local.twitter.com/libs-releases-local" - ;;"snapshots" "http://artifactory.local.twitter.com/libs-snapshots-local" - "conjars" "http://conjars.org/repo/"} + :repositories {"conjars" "http://conjars.org/repo/"} :dependencies [[com.twitter/kafka_2.9.2 "0.7.0" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] From c40f0ce165a421d2f88f3ce28ee0615dbf19ecad Mon Sep 17 00:00:00 2001 From: Jason Jackson Date: Thu, 10 Jan 2013 17:01:40 -0800 Subject: [PATCH 158/349] bumped version, upgraded storm dep --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 6e269dd354..413aa3c83e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject storm/storm-kafka "0.9.0-wip5-scala292" +(defproject storm/storm-kafka "0.9.0-wip15-scala292" :source-path "src/clj" :java-source-path "src/jvm" :javac-options {:debug "true" :fork "true"} @@ -6,7 +6,7 @@ :dependencies [[com.twitter/kafka_2.9.2 "0.7.0" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :dev-dependencies [[storm "0.9.0-wip9"] + :dev-dependencies [[storm "0.9.0-wip15"] [org.slf4j/log4j-over-slf4j "1.6.6"] [ch.qos.logback/logback-classic "1.0.6"] [org.clojure/clojure "1.4.0"]] From 1e00589634d1b1fbb9aafda0c50ba7cc2ccbc624 Mon Sep 17 00:00:00 2001 From: Nathan Marz Date: Fri, 11 Jan 2013 11:53:10 -0800 Subject: [PATCH 159/349] include offset when logging about committing offsets --- src/jvm/storm/kafka/PartitionManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 19e8ad99b0..f873e2db58 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -158,7 +158,7 @@ public void commit() { LOG.info("Wrote committed offset to ZK: " + committedTo); _committedTo = committedTo; } - LOG.info("Comitted offset for " + _partition); + LOG.info("Committed offset " + committedTo + " for " + _partition); } private String committedPath() { From b33e706503c50390d52c54032a95d955dd71d08a Mon Sep 17 00:00:00 2001 From: Trevor Wennblom Date: Mon, 14 Jan 2013 18:02:51 -0600 Subject: [PATCH 160/349] update to storm 0.8.2 --- m2-pom.xml | 4 +- project.clj | 2 +- .../storm/starter/trident/TridentReach.java | 61 ++++++++++--------- 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/m2-pom.xml b/m2-pom.xml index 984f171168..ba5c760fa5 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -56,7 +56,7 @@ storm storm - 0.8.1 + 0.8.2 provided @@ -96,7 +96,7 @@ - diff --git a/project.clj b/project.clj index 9fe650e390..6ff5fbd431 100644 --- a/project.clj +++ b/project.clj @@ -1,18 +1,27 @@ (defproject storm/storm-kafka-0.8-plus "0.1.0-SNAPSHOT" :description "Storm module for kafka > 0.8" + :plugins [[lein-junit "1.1.2"]] :java-source-paths ["src/jvm"] - :test-paths ["src/test/"] + :test-paths ["src/test"] + :junit ["src/test"] :repositories {"scala-tools" "http://scala-tools.org/repo-releases" - "conjars" "http://conjars.org/repo/"} + "conjars" "http://conjars.org/repo/"} :dependencies [[org.scala-lang/scala-library "2.9.2"] - [org.apache/kafka_2.9.2 "0.8.0-SNAPSHOT" + [junit/junit "4.11" :scope "test"] + [com.netflix.curator/curator-framework "1.3.2" + :exclusions [log4j/log4j + org.slf4j/slf4j-log4j12] ] + [com.netflix.curator/curator-recipes "1.2.6" + :exclusions [log4j/log4j] :scope "test"] + [com.netflix.curator/curator-test "1.2.6" + :exclusions [log4j/log4j] :scope "test"] + [org.apache/kafka_2.9.2 "0.8.0-SNAPSHOT" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] - :profiles - {:provided {:dependencies [[storm "0.9.0-wip15"] - [org.slf4j/log4j-over-slf4j "1.6.6"] - ;;[ch.qos.logback/logback-classic "1.0.6"] - [org.clojure/clojure "1.4.0"]]}} + :profiles {:provided {:dependencies [[storm "0.9.0-wip15"] + [org.slf4j/log4j-over-slf4j "1.6.6"] + ;;[ch.qos.logback/logback-classic "1.0.6"] + [org.clojure/clojure "1.4.0"]]}} :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] :min-lein-version "2.0") diff --git a/src/jvm/storm/kafka/DynamicBrokersReader.java b/src/jvm/storm/kafka/DynamicBrokersReader.java index 7d4fd58da9..9b9ad8e839 100644 --- a/src/jvm/storm/kafka/DynamicBrokersReader.java +++ b/src/jvm/storm/kafka/DynamicBrokersReader.java @@ -23,19 +23,15 @@ public class DynamicBrokersReader { private String _topic; public DynamicBrokersReader(Map conf, String zkStr, String zkPath, String topic) { - try { - _zkPath = zkPath; - _topic = topic; - _curator = CuratorFrameworkFactory.newClient( - zkStr, - Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), - 15000, - new RetryNTimes(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), - Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); - _curator.start(); - } catch (IOException e) { - throw new RuntimeException(e); - } + _zkPath = zkPath; + _topic = topic; + _curator = CuratorFrameworkFactory.newClient( + zkStr, + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), + 15000, + new RetryNTimes(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); + _curator.start(); } /** @@ -45,7 +41,7 @@ public GlobalPartitionInformation getBrokerInfo() { GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(); try { int numPartitionsForTopic = getNumPartitions(); - String brokerInfoPath = _zkPath + "/ids"; + String brokerInfoPath = brokerPath(); for (int partition = 0; partition < numPartitionsForTopic; partition++) { int leader = getLeaderFor(partition); String path = brokerInfoPath + "/" + leader; @@ -64,9 +60,11 @@ public GlobalPartitionInformation getBrokerInfo() { return globalPartitionInformation; } + + private int getNumPartitions() { try { - String topicBrokersPath = _zkPath + "/topics/" + _topic + "/partitions"; + String topicBrokersPath = partitionPath(); List children = _curator.getChildren().forPath(topicBrokersPath); return children.size(); } catch(Exception e) { @@ -74,6 +72,13 @@ private int getNumPartitions() { } } + public String partitionPath() { + return _zkPath + "/topics/" + _topic + "/partitions"; + } + + public String brokerPath() { + return _zkPath + "/ids"; + } /** * get /brokers/topics/distributedTopic/partitions/1/state @@ -83,7 +88,7 @@ private int getNumPartitions() { */ private int getLeaderFor(long partition) { try { - String topicBrokersPath = _zkPath + "/topics/" + _topic + "/partitions"; + String topicBrokersPath = partitionPath(); byte[] hostPortData = _curator.getData().forPath(topicBrokersPath + "/" + partition + "/state" ); Map value = (Map) JSONValue.parse(new String(hostPortData, "UTF-8")); Integer leader = ((Number) value.get("leader")).intValue(); diff --git a/src/test/storm/kafka/DynamicBrokersReaderTest.java b/src/test/storm/kafka/DynamicBrokersReaderTest.java new file mode 100644 index 0000000000..307edfdf71 --- /dev/null +++ b/src/test/storm/kafka/DynamicBrokersReaderTest.java @@ -0,0 +1,154 @@ +package storm.kafka; + +import backtype.storm.Config; +import com.netflix.curator.framework.CuratorFramework; +import com.netflix.curator.framework.CuratorFrameworkFactory; +import com.netflix.curator.retry.ExponentialBackoffRetry; +import com.netflix.curator.test.TestingServer; +import com.netflix.curator.utils.ZKPaths; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import storm.kafka.trident.GlobalPartitionInformation; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * User: thomas + * Date: 16/05/2013 + * Time: 20:35 + */ +public class DynamicBrokersReaderTest { + private DynamicBrokersReader dynamicBrokersReader; + private String masterPath = "/brokers"; + private String topic = "testing"; + private CuratorFramework zookeeper; + private TestingServer server; + + @Before + public void setUp() throws Exception { + server = new TestingServer(); + String connectionString = server.getConnectString(); + Map conf = new HashMap(); + conf.put(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT, 1000); + conf.put(Config.STORM_ZOOKEEPER_RETRY_TIMES, 4); + conf.put(Config.STORM_ZOOKEEPER_RETRY_INTERVAL, 5); + ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3); + zookeeper = CuratorFrameworkFactory.newClient(connectionString, retryPolicy); + dynamicBrokersReader = new DynamicBrokersReader(conf, connectionString, masterPath, topic); + zookeeper.start(); + } + + @After + public void tearDown() throws Exception { + server.close(); + } + + private void addPartition(int id, String host, int port) throws Exception { + writePartitionId(id); + writeLeader(id, 0); + writeLeaderDetails(0, host, port); + } + + private void addPartition(int id, int leader, String host, int port) throws Exception { + writePartitionId(id); + writeLeader(id, leader); + writeLeaderDetails(leader, host, port); + } + + private void writePartitionId(int id) throws Exception { + String path = dynamicBrokersReader.partitionPath(); + writeDataToPath(path, ("" + id)); + } + + private void writeDataToPath(String path, String data) throws Exception { + ZKPaths.mkdirs(zookeeper.getZookeeperClient().getZooKeeper(), path); + zookeeper.setData().forPath(path, data.getBytes()); + } + + private void writeLeader(int id, int leaderId) throws Exception { + String path = dynamicBrokersReader.partitionPath() + "/" + id + "/state"; + String value = " { \"controller_epoch\":4, \"isr\":[ 1, 0 ], \"leader\":" +leaderId + ", \"leader_epoch\":1, \"version\":1 }"; + writeDataToPath(path, value); + } + + private void writeLeaderDetails(int leaderId, String host, int port) throws Exception{ + String path = dynamicBrokersReader.brokerPath() + "/" + leaderId; + String value = "{ \"host\":\"" + host + "\", \"jmx_port\":9999, \"port\":" + port + ", \"version\":1 }"; + writeDataToPath(path, value); + } + + @Test + public void testGetBrokerInfo() throws Exception { + String host = "localhost"; + int port = 9092; + int partition = 0; + addPartition(partition, host, port); + GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); + assertEquals(1,brokerInfo.getOrderedPartitions().size()); + assertEquals(port, brokerInfo.getHostFor(partition).port); + assertEquals(host, brokerInfo.getHostFor(partition).host); + } + + + @Test + public void testMultiplePartitionsOnDifferentHosts() throws Exception { + String host = "localhost"; + int port = 9092; + int secondPort = 9093; + int partition = 0; + int secondPartition = partition + 1; + addPartition(partition, 0, host, port); + addPartition(secondPartition, 1, host, secondPort); + + GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); + assertEquals(2,brokerInfo.getOrderedPartitions().size()); + + assertEquals(port, brokerInfo.getHostFor(partition).port); + assertEquals(host, brokerInfo.getHostFor(partition).host); + + assertEquals(secondPort, brokerInfo.getHostFor(secondPartition).port); + assertEquals(host, brokerInfo.getHostFor(secondPartition).host); + } + + + @Test + public void testMultiplePartitionsOnSameHost() throws Exception { + String host = "localhost"; + int port = 9092; + int partition = 0; + int secondPartition = partition + 1; + addPartition(partition, 0, host, port); + addPartition(secondPartition, 0, host, port); + + GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); + assertEquals(2,brokerInfo.getOrderedPartitions().size()); + + assertEquals(port, brokerInfo.getHostFor(partition).port); + assertEquals(host, brokerInfo.getHostFor(partition).host); + + assertEquals(port, brokerInfo.getHostFor(secondPartition).port); + assertEquals(host, brokerInfo.getHostFor(secondPartition).host); + } + + @Test + public void testSwitchHostForPartition() throws Exception { + String host = "localhost"; + int port = 9092; + int partition = 0; + addPartition(partition, host, port); + GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); + assertEquals(port, brokerInfo.getHostFor(partition).port); + assertEquals(host, brokerInfo.getHostFor(partition).host); + + String newHost = host + "switch"; + int newPort = port + 1; + addPartition(partition, newHost, newPort); + brokerInfo = dynamicBrokersReader.getBrokerInfo(); + assertEquals(newPort, brokerInfo.getHostFor(partition).port); + assertEquals(newHost, brokerInfo.getHostFor(partition).host); + } +} From f63b276ea3e99ff7257a31b68e1b7a364bf9f1ac Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Thu, 16 May 2013 22:51:54 +0100 Subject: [PATCH 185/349] moved to maven --- project.clj => lein-project.clj | 0 pom.xml | 6 ------ 2 files changed, 6 deletions(-) rename project.clj => lein-project.clj (100%) diff --git a/project.clj b/lein-project.clj similarity index 100% rename from project.clj rename to lein-project.clj diff --git a/pom.xml b/pom.xml index 1a585aeab9..690978f063 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,6 @@ scm:git:git://github.com/wurstmeister/storm-kafka-0.8-plus.git scm:git:ssh://git@github.com/wurstmeister/storm-kafka-0.8-plus.git - 4024080de5ee9d131b766f870007b13d723ee3cc https://github.com/wurstmeister/storm-kafka-0.8-plus @@ -161,8 +160,3 @@ - - From f8088920b71b4ab704ffc76f63dab74dc1e834cd Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Wed, 22 May 2013 09:48:16 +0100 Subject: [PATCH 186/349] Refactored emitters * extracted emitter code to single class * downgraded curator version to avoid conflicts --- lein-project.clj | 8 +- pom.xml | 10 +- src/jvm/storm/kafka/DynamicBrokersReader.java | 8 +- src/jvm/storm/kafka/trident/KafkaUtils.java | 79 +----- .../trident/OpaqueTridentKafkaSpout.java | 76 +----- .../kafka/trident/StaticBrokerReader.java | 9 +- .../TransactionalTridentKafkaSpout.java | 90 +------ .../kafka/trident/TridentKafkaEmitter.java | 252 ++++++++++++++++++ .../storm/kafka/DynamicBrokersReaderTest.java | 1 - 9 files changed, 280 insertions(+), 253 deletions(-) create mode 100644 src/jvm/storm/kafka/trident/TridentKafkaEmitter.java diff --git a/lein-project.clj b/lein-project.clj index 6ff5fbd431..27f0d81c56 100644 --- a/lein-project.clj +++ b/lein-project.clj @@ -9,13 +9,13 @@ "conjars" "http://conjars.org/repo/"} :dependencies [[org.scala-lang/scala-library "2.9.2"] [junit/junit "4.11" :scope "test"] - [com.netflix.curator/curator-framework "1.3.2" + [com.netflix.curator/curator-framework "1.0.1" :exclusions [log4j/log4j org.slf4j/slf4j-log4j12] ] - [com.netflix.curator/curator-recipes "1.2.6" - :exclusions [log4j/log4j] :scope "test"] - [com.netflix.curator/curator-test "1.2.6" + [com.netflix.curator/curator-recipes "1.0.1" :exclusions [log4j/log4j] :scope "test"] + [com.netflix.curator/curator-test "1.0.1" + :exclusions [log4j/log4j org.testng/testng] :scope "test"] [org.apache/kafka_2.9.2 "0.8.0-SNAPSHOT" :exclusions [org.apache.zookeeper/zookeeper log4j/log4j]]] diff --git a/pom.xml b/pom.xml index 690978f063..85c39faaef 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ com.netflix.curator curator-framework - 1.3.2 + 1.0.1 log4j @@ -104,7 +104,7 @@ com.netflix.curator curator-recipes - 1.2.6 + 1.0.1 log4j @@ -116,12 +116,16 @@ com.netflix.curator curator-test - 1.2.6 + 1.0.1 log4j log4j + + org.testng + testng + test diff --git a/src/jvm/storm/kafka/DynamicBrokersReader.java b/src/jvm/storm/kafka/DynamicBrokersReader.java index 9b9ad8e839..ae155343b9 100644 --- a/src/jvm/storm/kafka/DynamicBrokersReader.java +++ b/src/jvm/storm/kafka/DynamicBrokersReader.java @@ -25,13 +25,17 @@ public class DynamicBrokersReader { public DynamicBrokersReader(Map conf, String zkStr, String zkPath, String topic) { _zkPath = zkPath; _topic = topic; - _curator = CuratorFrameworkFactory.newClient( + try { + _curator = CuratorFrameworkFactory.newClient( zkStr, Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), 15000, new RetryNTimes(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); - _curator.start(); + _curator.start(); + } catch (IOException ex) { + LOG.error("can't connect to zookeeper"); + } } /** diff --git a/src/jvm/storm/kafka/trident/KafkaUtils.java b/src/jvm/storm/kafka/trident/KafkaUtils.java index 94495590e6..ccbfc3a433 100644 --- a/src/jvm/storm/kafka/trident/KafkaUtils.java +++ b/src/jvm/storm/kafka/trident/KafkaUtils.java @@ -1,26 +1,18 @@ package storm.kafka.trident; -import backtype.storm.metric.api.CombinedMetric; import backtype.storm.metric.api.IMetric; -import backtype.storm.metric.api.ReducedMetric; -import backtype.storm.utils.Utils; -import com.google.common.collect.ImmutableMap; -import kafka.api.FetchRequest; -import kafka.api.FetchRequestBuilder; import kafka.api.PartitionOffsetRequestInfo; import kafka.common.TopicAndPartition; import kafka.javaapi.OffsetRequest; import kafka.javaapi.consumer.SimpleConsumer; -import kafka.javaapi.message.ByteBufferMessageSet; -import kafka.message.Message; -import kafka.message.MessageAndOffset; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import storm.kafka.*; -import storm.trident.operation.TridentCollector; -import java.net.ConnectException; -import java.util.*; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; public class KafkaUtils { public static final Logger LOG = LoggerFactory.getLogger(KafkaUtils.class); @@ -35,69 +27,6 @@ public static IBrokerReader makeBrokerReader(Map stormConf, KafkaConfig conf) { } } - - public static Map emitPartitionBatchNew(TridentKafkaConfig config, SimpleConsumer consumer, GlobalPartitionId partition, TridentCollector collector, Map lastMeta, String topologyInstanceId, String topologyName, - ReducedMetric meanMetric, CombinedMetric maxMetric) { - long offset; - if(lastMeta!=null) { - String lastInstanceId = null; - Map lastTopoMeta = (Map) lastMeta.get("topology"); - if(lastTopoMeta!=null) { - lastInstanceId = (String) lastTopoMeta.get("id"); - } - if(config.forceFromStart && !topologyInstanceId.equals(lastInstanceId)) { - offset = getOffset(consumer, config.topic, partition.partition, config.startOffsetTime); - } else { - offset = (Long) lastMeta.get("nextOffset"); - } - } else { - long startTime = -1; - if(config.forceFromStart) startTime = config.startOffsetTime; - offset = getOffset(consumer, config.topic, partition.partition, startTime); - } - ByteBufferMessageSet msgs; - try { - long start = System.nanoTime(); - FetchRequestBuilder builder = new FetchRequestBuilder(); - FetchRequest fetchRequest = builder.addFetch(config.topic, partition.partition, offset, config.fetchSizeBytes).build(); - msgs = consumer.fetch(fetchRequest).messageSet(config.topic, partition.partition); - long end = System.nanoTime(); - long millis = (end - start) / 1000000; - meanMetric.update(millis); - maxMetric.update(millis); - } catch(Exception e) { - if(e instanceof ConnectException) { - throw new FailedFetchException(e); - } else { - throw new RuntimeException(e); - } - } - long endoffset = offset; - for(MessageAndOffset msg: msgs) { - emit(config, collector, msg.message()); - endoffset = msg.nextOffset(); - } - Map newMeta = new HashMap(); - newMeta.put("offset", offset); - newMeta.put("nextOffset", endoffset); - newMeta.put("instanceId", topologyInstanceId); - newMeta.put("partition", partition.partition); - newMeta.put("broker", ImmutableMap.of("host", partition.host.host, "port", partition.host.port)); - newMeta.put("topic", config.topic); - newMeta.put("topology", ImmutableMap.of("name", topologyName, "id", topologyInstanceId)); - return newMeta; - } - - public static void emit(TridentKafkaConfig config, TridentCollector collector, Message msg) { - Iterable> values = - config.scheme.deserialize(Utils.toByteArray(msg.payload())); - if(values!=null) { - for(List value: values) - collector.emit(value); - } - } - - public static long getOffset(SimpleConsumer consumer, String topic, int partition, long startOffsetTime) { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map requestInfo = new HashMap(); diff --git a/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java b/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java index 550195b823..087c7b9fe8 100644 --- a/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java +++ b/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java @@ -1,26 +1,16 @@ package storm.kafka.trident; -import backtype.storm.Config; -import backtype.storm.metric.api.CombinedMetric; -import backtype.storm.metric.api.MeanReducer; -import backtype.storm.metric.api.ReducedMetric; import backtype.storm.task.TopologyContext; import backtype.storm.tuple.Fields; -import com.google.common.collect.ImmutableMap; -import kafka.javaapi.consumer.SimpleConsumer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import storm.kafka.DynamicPartitionConnections; import storm.kafka.GlobalPartitionId; -import storm.trident.operation.TridentCollector; import storm.trident.spout.IOpaquePartitionedTridentSpout; -import storm.trident.topology.TransactionAttempt; -import java.util.*; +import java.util.Map; +import java.util.UUID; public class OpaqueTridentKafkaSpout implements IOpaquePartitionedTridentSpout { - public static final Logger LOG = LoggerFactory.getLogger(OpaqueTridentKafkaSpout.class); + TridentKafkaConfig _config; String _topologyInstanceId = UUID.randomUUID().toString(); @@ -31,7 +21,7 @@ public OpaqueTridentKafkaSpout(TridentKafkaConfig config) { @Override public IOpaquePartitionedTridentSpout.Emitter getEmitter(Map conf, TopologyContext context) { - return new Emitter(conf, context); + return new TridentKafkaEmitter(conf, context, _config, _topologyInstanceId).asOpaqueEmitter(); } @Override @@ -49,62 +39,4 @@ public Map getComponentConfiguration() { return null; } - - class Emitter implements IOpaquePartitionedTridentSpout.Emitter { - DynamicPartitionConnections _connections; - String _topologyName; - KafkaUtils.KafkaOffsetMetric _kafkaOffsetMetric; - ReducedMetric _kafkaMeanFetchLatencyMetric; - CombinedMetric _kafkaMaxFetchLatencyMetric; - - public Emitter(Map conf, TopologyContext context) { - _connections = new DynamicPartitionConnections(_config, KafkaUtils.makeBrokerReader(conf, _config)); - _topologyName = (String) conf.get(Config.TOPOLOGY_NAME); - _kafkaOffsetMetric = new KafkaUtils.KafkaOffsetMetric(_config.topic, _connections); - context.registerMetric("kafkaOffset", _kafkaOffsetMetric, 60); - _kafkaMeanFetchLatencyMetric = context.registerMetric("kafkaFetchAvg", new MeanReducer(), 60); - _kafkaMaxFetchLatencyMetric = context.registerMetric("kafkaFetchMax", new MaxMetric(), 60); - } - - @Override - public Map emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map lastMeta) { - try { - SimpleConsumer consumer = _connections.register(partition); - Map ret = KafkaUtils.emitPartitionBatchNew(_config, consumer, partition, collector, lastMeta, _topologyInstanceId, _topologyName, _kafkaMeanFetchLatencyMetric, _kafkaMaxFetchLatencyMetric); - _kafkaOffsetMetric.setLatestEmittedOffset(partition, (Long)ret.get("offset")); - return ret; - } catch(FailedFetchException e) { - LOG.warn("Failed to fetch from partition " + partition); - if(lastMeta==null) { - return null; - } else { - Map ret = new HashMap(); - ret.put("offset", lastMeta.get("nextOffset")); - ret.put("nextOffset", lastMeta.get("nextOffset")); - ret.put("partition", partition.partition); - ret.put("broker", ImmutableMap.of("host", partition.host.host, "port", partition.host.port)); - ret.put("topic", _config.topic); - ret.put("topology", ImmutableMap.of("name", _topologyName, "id", _topologyInstanceId)); - - return ret; - } - } - } - - @Override - public void close() { - _connections.clear(); - } - - @Override - public List getOrderedPartitions(GlobalPartitionInformation partitions) { - return partitions.getOrderedPartitions(); - } - - @Override - public void refreshPartitions(List list) { - _connections.clear(); - _kafkaOffsetMetric.refreshPartitions(new HashSet(list)); - } - } } diff --git a/src/jvm/storm/kafka/trident/StaticBrokerReader.java b/src/jvm/storm/kafka/trident/StaticBrokerReader.java index ced411c402..dabbd5e7a0 100644 --- a/src/jvm/storm/kafka/trident/StaticBrokerReader.java +++ b/src/jvm/storm/kafka/trident/StaticBrokerReader.java @@ -1,15 +1,8 @@ package storm.kafka.trident; -import storm.kafka.GlobalPartitionId; -import storm.kafka.HostPort; -import storm.kafka.StaticHosts; - -import java.util.List; - - public class StaticBrokerReader implements IBrokerReader { - GlobalPartitionInformation brokers = new GlobalPartitionInformation(); + private GlobalPartitionInformation brokers = new GlobalPartitionInformation(); public StaticBrokerReader(GlobalPartitionInformation partitionInformation) { this.brokers = partitionInformation; diff --git a/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java b/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java index d4b4d81121..efe8075eed 100644 --- a/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java +++ b/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java @@ -1,23 +1,10 @@ package storm.kafka.trident; -import backtype.storm.Config; -import backtype.storm.metric.api.CombinedMetric; -import backtype.storm.metric.api.MeanReducer; -import backtype.storm.metric.api.ReducedMetric; import backtype.storm.task.TopologyContext; import backtype.storm.tuple.Fields; -import kafka.api.FetchRequestBuilder; -import kafka.javaapi.consumer.SimpleConsumer; -import kafka.javaapi.message.ByteBufferMessageSet; -import kafka.message.MessageAndOffset; -import storm.kafka.DynamicPartitionConnections; import storm.kafka.GlobalPartitionId; -import storm.trident.operation.TridentCollector; import storm.trident.spout.IPartitionedTridentSpout; -import storm.trident.topology.TransactionAttempt; -import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.UUID; @@ -30,80 +17,7 @@ public class TransactionalTridentKafkaSpout implements IPartitionedTridentSpout< public TransactionalTridentKafkaSpout(TridentKafkaConfig config) { _config = config; } - - - - class Emitter implements IPartitionedTridentSpout.Emitter { - DynamicPartitionConnections _connections; - String _topologyName; - TopologyContext _context; - KafkaUtils.KafkaOffsetMetric _kafkaOffsetMetric; - ReducedMetric _kafkaMeanFetchLatencyMetric; - CombinedMetric _kafkaMaxFetchLatencyMetric; - - - public Emitter(Map conf, TopologyContext context) { - IBrokerReader brokerReader = KafkaUtils.makeBrokerReader(conf, _config); - _connections = new DynamicPartitionConnections(_config, brokerReader); - _topologyName = (String) conf.get(Config.TOPOLOGY_NAME); - _context = context; - _kafkaOffsetMetric = new KafkaUtils.KafkaOffsetMetric(_config.topic, _connections); - context.registerMetric("kafkaOffset", _kafkaOffsetMetric, 60); - _kafkaMeanFetchLatencyMetric = context.registerMetric("kafkaFetchAvg", new MeanReducer(), 60); - _kafkaMaxFetchLatencyMetric = context.registerMetric("kafkaFetchMax", new MaxMetric(), 60); - - } - - @Override - public Map emitPartitionBatchNew(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map lastMeta) { - SimpleConsumer consumer = _connections.register(partition); - Map ret = KafkaUtils.emitPartitionBatchNew(_config, consumer, partition, collector, lastMeta, _topologyInstanceId, _topologyName, _kafkaMeanFetchLatencyMetric, _kafkaMaxFetchLatencyMetric); - _kafkaOffsetMetric.setLatestEmittedOffset(partition, (Long)ret.get("offset")); - return ret; - } - @Override - public void emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map meta) { - String instanceId = (String) meta.get("instanceId"); - if(!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) { - SimpleConsumer consumer = _connections.register(partition); - long offset = (Long) meta.get("offset"); - long nextOffset = (Long) meta.get("nextOffset"); - long start = System.nanoTime(); - ByteBufferMessageSet msgs = consumer.fetch(new FetchRequestBuilder().addFetch(_config.topic, partition.partition, offset, _config.fetchSizeBytes).build()).messageSet(_config.topic, partition.partition); - long end = System.nanoTime(); - long millis = (end - start) / 1000000; - _kafkaMeanFetchLatencyMetric.update(millis); - _kafkaMaxFetchLatencyMetric.update(millis); - - for(MessageAndOffset msg: msgs) { - if(offset == nextOffset) break; - if(offset > nextOffset) { - throw new RuntimeException("Error when re-emitting batch. overshot the end offset"); - } - KafkaUtils.emit(_config, collector, msg.message()); - offset = msg.nextOffset(); - } - } - } - - @Override - public void close() { - _connections.clear(); - } - - @Override - public List getOrderedPartitions(GlobalPartitionInformation partitions) { - return partitions.getOrderedPartitions(); - } - - @Override - public void refreshPartitions(List list) { - _connections.clear(); - _kafkaOffsetMetric.refreshPartitions(new HashSet(list)); - } - } - @Override public IPartitionedTridentSpout.Coordinator getCoordinator(Map conf, TopologyContext context) { @@ -112,12 +26,12 @@ public IPartitionedTridentSpout.Coordinator getCoordinator(Map conf, TopologyCon @Override public IPartitionedTridentSpout.Emitter getEmitter(Map conf, TopologyContext context) { - return new Emitter(conf, context); + return new TridentKafkaEmitter(conf, context, _config, _topologyInstanceId).asTransactionalEmitter(); } @Override public Fields getOutputFields() { - return _config.scheme.getOutputFields(); + return _config.scheme.getOutputFields(); } @Override diff --git a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java new file mode 100644 index 0000000000..6c7360c0a7 --- /dev/null +++ b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java @@ -0,0 +1,252 @@ +package storm.kafka.trident; + +import backtype.storm.Config; +import backtype.storm.metric.api.CombinedMetric; +import backtype.storm.metric.api.MeanReducer; +import backtype.storm.metric.api.ReducedMetric; +import backtype.storm.task.TopologyContext; +import backtype.storm.utils.Utils; +import com.google.common.collect.ImmutableMap; +import kafka.api.FetchRequest; +import kafka.api.FetchRequestBuilder; +import kafka.javaapi.consumer.SimpleConsumer; +import kafka.javaapi.message.ByteBufferMessageSet; +import kafka.message.Message; +import kafka.message.MessageAndOffset; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import storm.kafka.DynamicPartitionConnections; +import storm.kafka.GlobalPartitionId; +import storm.trident.operation.TridentCollector; +import storm.trident.spout.IOpaquePartitionedTridentSpout; +import storm.trident.spout.IPartitionedTridentSpout; +import storm.trident.topology.TransactionAttempt; + +import java.net.ConnectException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +/** + * Date: 21/05/2013 + * Time: 08:38 + */ +public class TridentKafkaEmitter { + + public static final Logger LOG = LoggerFactory.getLogger(TridentKafkaEmitter.class); + + private DynamicPartitionConnections _connections; + private String _topologyName; + private KafkaUtils.KafkaOffsetMetric _kafkaOffsetMetric; + private ReducedMetric _kafkaMeanFetchLatencyMetric; + private CombinedMetric _kafkaMaxFetchLatencyMetric; + private TridentKafkaConfig _config; + private String _topologyInstanceId; + + public TridentKafkaEmitter(Map conf, TopologyContext context, TridentKafkaConfig config, String topologyInstanceId) { + _config = config; + _topologyInstanceId = topologyInstanceId; + _connections = new DynamicPartitionConnections(_config, KafkaUtils.makeBrokerReader(conf, _config)); + _topologyName = (String) conf.get(Config.TOPOLOGY_NAME); + _kafkaOffsetMetric = new KafkaUtils.KafkaOffsetMetric(_config.topic, _connections); + context.registerMetric("kafkaOffset", _kafkaOffsetMetric, 60); + _kafkaMeanFetchLatencyMetric = context.registerMetric("kafkaFetchAvg", new MeanReducer(), 60); + _kafkaMaxFetchLatencyMetric = context.registerMetric("kafkaFetchMax", new MaxMetric(), 60); + } + + + private Map failFastEmitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map lastMeta) { + SimpleConsumer consumer = _connections.register(partition); + Map ret = doEmitNewPartitionBatch(consumer, partition, collector, lastMeta); + _kafkaOffsetMetric.setLatestEmittedOffset(partition, (Long) ret.get("offset")); + return ret; + } + + private Map emitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map lastMeta) { + try { + return failFastEmitNewPartitionBatch(attempt, collector, partition, lastMeta); + } catch (FailedFetchException e) { + LOG.warn("Failed to fetch from partition " + partition); + if (lastMeta == null) { + return null; + } else { + Map ret = new HashMap(); + ret.put("offset", lastMeta.get("nextOffset")); + ret.put("nextOffset", lastMeta.get("nextOffset")); + ret.put("partition", partition.partition); + ret.put("broker", ImmutableMap.of("host", partition.host.host, "port", partition.host.port)); + ret.put("topic", _config.topic); + ret.put("topology", ImmutableMap.of("name", _topologyName, "id", _topologyInstanceId)); + return ret; + } + } + } + + private Map doEmitNewPartitionBatch(SimpleConsumer consumer, GlobalPartitionId partition, TridentCollector collector, Map lastMeta) { + long offset; + if (lastMeta != null) { + String lastInstanceId = null; + Map lastTopoMeta = (Map) lastMeta.get("topology"); + if (lastTopoMeta != null) { + lastInstanceId = (String) lastTopoMeta.get("id"); + } + if (_config.forceFromStart && !_topologyInstanceId.equals(lastInstanceId)) { + offset = KafkaUtils.getOffset(consumer, _config.topic, partition.partition, _config.startOffsetTime); + } else { + offset = (Long) lastMeta.get("nextOffset"); + } + } else { + long startTime = -1; + if (_config.forceFromStart) startTime = _config.startOffsetTime; + offset = KafkaUtils.getOffset(consumer, _config.topic, partition.partition, startTime); + } + ByteBufferMessageSet msgs; + try { + long start = System.nanoTime(); + FetchRequestBuilder builder = new FetchRequestBuilder(); + FetchRequest fetchRequest = builder.addFetch(_config.topic, partition.partition, offset, _config.fetchSizeBytes).build(); + msgs = consumer.fetch(fetchRequest).messageSet(_config.topic, partition.partition); + long end = System.nanoTime(); + long millis = (end - start) / 1000000; + _kafkaMeanFetchLatencyMetric.update(millis); + _kafkaMaxFetchLatencyMetric.update(millis); + } catch (Exception e) { + if (e instanceof ConnectException) { + throw new FailedFetchException(e); + } else { + throw new RuntimeException(e); + } + } + long endoffset = offset; + for (MessageAndOffset msg : msgs) { + emit(collector, msg.message()); + endoffset = msg.nextOffset(); + } + Map newMeta = new HashMap(); + newMeta.put("offset", offset); + newMeta.put("nextOffset", endoffset); + newMeta.put("instanceId", _topologyInstanceId); + newMeta.put("partition", partition.partition); + newMeta.put("broker", ImmutableMap.of("host", partition.host.host, "port", partition.host.port)); + newMeta.put("topic", _config.topic); + newMeta.put("topology", ImmutableMap.of("name", _topologyName, "id", _topologyInstanceId)); + return newMeta; + } + + /** + * re-emit the batch described by the meta data provided + * + * @param attempt + * @param collector + * @param partition + * @param meta + */ + private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map meta) { + LOG.info("re-emitting batch, attempt " + attempt); + String instanceId = (String) meta.get("instanceId"); + if (!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) { + SimpleConsumer consumer = _connections.register(partition); + long offset = (Long) meta.get("offset"); + long nextOffset = (Long) meta.get("nextOffset"); + long start = System.nanoTime(); + ByteBufferMessageSet msgs = consumer.fetch(new FetchRequestBuilder().addFetch(_config.topic, partition.partition, offset, _config.fetchSizeBytes).build()).messageSet(_config.topic, partition.partition); + long end = System.nanoTime(); + long millis = (end - start) / 1000000; + _kafkaMeanFetchLatencyMetric.update(millis); + _kafkaMaxFetchLatencyMetric.update(millis); + + for (MessageAndOffset msg : msgs) { + if (offset == nextOffset) break; + if (offset > nextOffset) { + throw new RuntimeException("Error when re-emitting batch. overshot the end offset"); + } + emit(collector, msg.message()); + offset = msg.nextOffset(); + } + } + } + + private void emit(TridentCollector collector, Message msg) { + Iterable> values = + _config.scheme.deserialize(Utils.toByteArray(msg.payload())); + if (values != null) { + for (List value : values) { + collector.emit(value); + } + } + } + + private void clear() { + _connections.clear(); + } + + private List orderPartitions(GlobalPartitionInformation partitions) { + return partitions.getOrderedPartitions(); + } + + private void refresh(List list) { + _connections.clear(); + _kafkaOffsetMetric.refreshPartitions(new HashSet(list)); + } + + + public IOpaquePartitionedTridentSpout.Emitter asOpaqueEmitter() { + + return new IOpaquePartitionedTridentSpout.Emitter() { + + @Override + public Map emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, GlobalPartitionId globalPartitionId, Map map) { + return emitNewPartitionBatch(transactionAttempt, tridentCollector, globalPartitionId, map); + } + + @Override + public void refreshPartitions(List globalPartitionIds) { + refresh(globalPartitionIds); + } + + @Override + public List getOrderedPartitions(GlobalPartitionInformation partitionInformation) { + return orderPartitions(partitionInformation); + } + + @Override + public void close() { + clear(); + } + }; + } + + public IPartitionedTridentSpout.Emitter asTransactionalEmitter() { + return new IPartitionedTridentSpout.Emitter() { + + @Override + public Map emitPartitionBatchNew(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, GlobalPartitionId globalPartitionId, Map map) { + return failFastEmitNewPartitionBatch(transactionAttempt, tridentCollector, globalPartitionId, map); + } + + @Override + public void emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, GlobalPartitionId globalPartitionId, Map map) { + reEmitPartitionBatch(transactionAttempt, tridentCollector, globalPartitionId, map); + } + + @Override + public void refreshPartitions(List globalPartitionIds) { + refresh(globalPartitionIds); + } + + @Override + public List getOrderedPartitions(GlobalPartitionInformation partitionInformation) { + return orderPartitions(partitionInformation); + } + + @Override + public void close() { + clear(); + } + }; + + } + + +} diff --git a/src/test/storm/kafka/DynamicBrokersReaderTest.java b/src/test/storm/kafka/DynamicBrokersReaderTest.java index 307edfdf71..a6c2309b08 100644 --- a/src/test/storm/kafka/DynamicBrokersReaderTest.java +++ b/src/test/storm/kafka/DynamicBrokersReaderTest.java @@ -17,7 +17,6 @@ import static org.junit.Assert.assertEquals; /** - * User: thomas * Date: 16/05/2013 * Time: 20:35 */ From ddee8961af3d8618610320ee5793547879c91cfa Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Wed, 22 May 2013 11:43:36 +0100 Subject: [PATCH 187/349] added support for client id * removed unused leiningen project file --- lein-project.clj | 27 ---------------- .../kafka/DynamicPartitionConnections.java | 2 +- src/jvm/storm/kafka/KafkaConfig.java | 17 +++++++--- src/jvm/storm/kafka/PartitionManager.java | 2 +- .../kafka/StaticPartitionConnections.java | 2 +- src/jvm/storm/kafka/trident/KafkaUtils.java | 2 +- .../kafka/trident/TridentKafkaConfig.java | 11 +++++-- .../kafka/trident/TridentKafkaEmitter.java | 32 +++++++++---------- 8 files changed, 41 insertions(+), 54 deletions(-) delete mode 100644 lein-project.clj diff --git a/lein-project.clj b/lein-project.clj deleted file mode 100644 index 27f0d81c56..0000000000 --- a/lein-project.clj +++ /dev/null @@ -1,27 +0,0 @@ -(defproject storm/storm-kafka-0.8-plus "0.1.0-SNAPSHOT" - :description "Storm module for kafka > 0.8" - :plugins [[lein-junit "1.1.2"]] - :java-source-paths ["src/jvm"] - :test-paths ["src/test"] - :junit ["src/test"] - - :repositories {"scala-tools" "http://scala-tools.org/repo-releases" - "conjars" "http://conjars.org/repo/"} - :dependencies [[org.scala-lang/scala-library "2.9.2"] - [junit/junit "4.11" :scope "test"] - [com.netflix.curator/curator-framework "1.0.1" - :exclusions [log4j/log4j - org.slf4j/slf4j-log4j12] ] - [com.netflix.curator/curator-recipes "1.0.1" - :exclusions [log4j/log4j] :scope "test"] - [com.netflix.curator/curator-test "1.0.1" - :exclusions [log4j/log4j org.testng/testng] :scope "test"] - [org.apache/kafka_2.9.2 "0.8.0-SNAPSHOT" - :exclusions [org.apache.zookeeper/zookeeper - log4j/log4j]]] - :profiles {:provided {:dependencies [[storm "0.9.0-wip15"] - [org.slf4j/log4j-over-slf4j "1.6.6"] - ;;[ch.qos.logback/logback-classic "1.0.6"] - [org.clojure/clojure "1.4.0"]]}} - :jvm-opts ["-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib"] - :min-lein-version "2.0") diff --git a/src/jvm/storm/kafka/DynamicPartitionConnections.java b/src/jvm/storm/kafka/DynamicPartitionConnections.java index c8a9c3925f..a5dd1cb332 100644 --- a/src/jvm/storm/kafka/DynamicPartitionConnections.java +++ b/src/jvm/storm/kafka/DynamicPartitionConnections.java @@ -40,7 +40,7 @@ public SimpleConsumer register(GlobalPartitionId id) { public SimpleConsumer register(HostPort host, int partition) { if(!_connections.containsKey(host)) { - _connections.put(host, new ConnectionInfo(new SimpleConsumer(host.host, host.port, _config.socketTimeoutMs, _config.bufferSizeBytes, kafka.api.OffsetRequest.DefaultClientId()))); + _connections.put(host, new ConnectionInfo(new SimpleConsumer(host.host, host.port, _config.socketTimeoutMs, _config.bufferSizeBytes, _config.clientId))); } ConnectionInfo info = _connections.get(host); info.partitions.add(partition); diff --git a/src/jvm/storm/kafka/KafkaConfig.java b/src/jvm/storm/kafka/KafkaConfig.java index a1675a9307..457eeb4620 100644 --- a/src/jvm/storm/kafka/KafkaConfig.java +++ b/src/jvm/storm/kafka/KafkaConfig.java @@ -7,20 +7,27 @@ public class KafkaConfig implements Serializable { - public BrokerHosts hosts; + public final BrokerHosts hosts; + public final String topic; + public final String clientId; + public int fetchSizeBytes = 1024*1024; public int socketTimeoutMs = 10000; public int bufferSizeBytes = 1024*1024; public MultiScheme scheme = new RawMultiScheme(); - public String topic; - public long startOffsetTime = -2; public boolean forceFromStart = false; + public long startOffsetTime = kafka.api.OffsetRequest.EarliestTime(); public KafkaConfig(BrokerHosts hosts, String topic) { - this.hosts = hosts; - this.topic = topic; + this(hosts, topic, kafka.api.OffsetRequest.DefaultClientId()); } + public KafkaConfig(BrokerHosts hosts, String topic, String clientId) { + this.hosts = hosts; + this.topic = topic; + this.clientId = clientId; + } + public void forceStartOffsetTime(long millis) { startOffsetTime = millis; forceFromStart = true; diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 39729a9a3b..9186953c6a 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -78,7 +78,7 @@ public PartitionManager(DynamicPartitionConnections connections, String topology _committedTo = KafkaUtils.getOffset(_consumer, spoutConfig.topic, id.partition, spoutConfig.startOffsetTime); LOG.info("Using startOffsetTime to choose last commit offset."); } else if(jsonTopologyId == null || jsonOffset == null) { // failed to parse JSON? - _committedTo = KafkaUtils.getOffset(_consumer, spoutConfig.topic, id.partition, -1); + _committedTo = KafkaUtils.getOffset(_consumer, spoutConfig.topic, id.partition, kafka.api.OffsetRequest.LatestTime()); LOG.info("Setting last commit offset to HEAD."); } else { _committedTo = jsonOffset; diff --git a/src/jvm/storm/kafka/StaticPartitionConnections.java b/src/jvm/storm/kafka/StaticPartitionConnections.java index 1998d87d89..2d40c8b67b 100644 --- a/src/jvm/storm/kafka/StaticPartitionConnections.java +++ b/src/jvm/storm/kafka/StaticPartitionConnections.java @@ -21,7 +21,7 @@ public StaticPartitionConnections(KafkaConfig conf) { public SimpleConsumer getConsumer(int partition) { if(!_kafka.containsKey(partition)) { HostPort hp = hosts.getPartitionInformation().getHostFor(partition); - _kafka.put(partition, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes, kafka.api.OffsetRequest.DefaultClientId())); + _kafka.put(partition, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes, _config.clientId)); } return _kafka.get(partition); diff --git a/src/jvm/storm/kafka/trident/KafkaUtils.java b/src/jvm/storm/kafka/trident/KafkaUtils.java index ccbfc3a433..585554cee4 100644 --- a/src/jvm/storm/kafka/trident/KafkaUtils.java +++ b/src/jvm/storm/kafka/trident/KafkaUtils.java @@ -32,7 +32,7 @@ public static long getOffset(SimpleConsumer consumer, String topic, int partitio Map requestInfo = new HashMap(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1)); OffsetRequest request = new OffsetRequest( - requestInfo, kafka.api.OffsetRequest.CurrentVersion(), kafka.api.OffsetRequest.DefaultClientId()); + requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); long[] offsets = consumer.getOffsetsBefore(request).offsets(topic, partition); if ( offsets.length > 0) { diff --git a/src/jvm/storm/kafka/trident/TridentKafkaConfig.java b/src/jvm/storm/kafka/trident/TridentKafkaConfig.java index ab308408bb..71955009c1 100644 --- a/src/jvm/storm/kafka/trident/TridentKafkaConfig.java +++ b/src/jvm/storm/kafka/trident/TridentKafkaConfig.java @@ -5,9 +5,16 @@ public class TridentKafkaConfig extends KafkaConfig { + + + public final IBatchCoordinator coordinator = new DefaultCoordinator(); + public TridentKafkaConfig(BrokerHosts hosts, String topic) { super(hosts, topic); } - - public IBatchCoordinator coordinator = new DefaultCoordinator(); + + public TridentKafkaConfig(BrokerHosts hosts, String topic, String clientId) { + super(hosts, topic, clientId); + } + } diff --git a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java index 6c7360c0a7..5ef88d2ded 100644 --- a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java +++ b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java @@ -97,20 +97,13 @@ private Map doEmitNewPartitionBatch(SimpleConsumer consumer, GlobalPartitionId p offset = (Long) lastMeta.get("nextOffset"); } } else { - long startTime = -1; + long startTime = kafka.api.OffsetRequest.LatestTime(); if (_config.forceFromStart) startTime = _config.startOffsetTime; offset = KafkaUtils.getOffset(consumer, _config.topic, partition.partition, startTime); } ByteBufferMessageSet msgs; try { - long start = System.nanoTime(); - FetchRequestBuilder builder = new FetchRequestBuilder(); - FetchRequest fetchRequest = builder.addFetch(_config.topic, partition.partition, offset, _config.fetchSizeBytes).build(); - msgs = consumer.fetch(fetchRequest).messageSet(_config.topic, partition.partition); - long end = System.nanoTime(); - long millis = (end - start) / 1000000; - _kafkaMeanFetchLatencyMetric.update(millis); - _kafkaMaxFetchLatencyMetric.update(millis); + msgs = fetchMessages(consumer, partition, offset); } catch (Exception e) { if (e instanceof ConnectException) { throw new FailedFetchException(e); @@ -134,6 +127,19 @@ private Map doEmitNewPartitionBatch(SimpleConsumer consumer, GlobalPartitionId p return newMeta; } + private ByteBufferMessageSet fetchMessages(SimpleConsumer consumer, GlobalPartitionId partition, long offset) { + ByteBufferMessageSet msgs; + long start = System.nanoTime(); + FetchRequestBuilder builder = new FetchRequestBuilder(); + FetchRequest fetchRequest = builder.addFetch(_config.topic, partition.partition, offset, _config.fetchSizeBytes).clientId(_config.clientId).build(); + msgs = consumer.fetch(fetchRequest).messageSet(_config.topic, partition.partition); + long end = System.nanoTime(); + long millis = (end - start) / 1000000; + _kafkaMeanFetchLatencyMetric.update(millis); + _kafkaMaxFetchLatencyMetric.update(millis); + return msgs; + } + /** * re-emit the batch described by the meta data provided * @@ -149,13 +155,7 @@ private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector c SimpleConsumer consumer = _connections.register(partition); long offset = (Long) meta.get("offset"); long nextOffset = (Long) meta.get("nextOffset"); - long start = System.nanoTime(); - ByteBufferMessageSet msgs = consumer.fetch(new FetchRequestBuilder().addFetch(_config.topic, partition.partition, offset, _config.fetchSizeBytes).build()).messageSet(_config.topic, partition.partition); - long end = System.nanoTime(); - long millis = (end - start) / 1000000; - _kafkaMeanFetchLatencyMetric.update(millis); - _kafkaMaxFetchLatencyMetric.update(millis); - + ByteBufferMessageSet msgs = fetchMessages(consumer, partition, offset); for (MessageAndOffset msg : msgs) { if (offset == nextOffset) break; if (offset > nextOffset) { From 83f183becb4f6bd45822beac48d9b6425be7391b Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Wed, 31 Jul 2013 22:06:00 +0100 Subject: [PATCH 188/349] Update to kafka 0.8 beta1 * updated pom and build instructions --- README.md | 21 ++++++++++++--------- pom.xml | 4 ++-- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c4cf602435..d37fb7f222 100644 --- a/README.md +++ b/README.md @@ -3,23 +3,26 @@ storm-kafka-0.8-plus Port of storm-kafka to support kafka >= 0.8 -I'm new to storm and kafka, so this is very much work in progress. - ## Installing kafka 0.8 into local maven repository -Until kafka 0.8 has been release you need to install kafka 0.8 into you local repository if you want to use it from maven +Unfortunately the current beta pom is slightly broken ([KAFKA-974](https://issues.apache.org/jira/browse/KAFKA-974)) so you might want to build your own - checkout kafka 0.8 branch - +- Check [KAFKA-939](https://issues.apache.org/jira/browse/KAFKA-939) for instructions on how to handle javadoc erros when publishing +- GPG: + - ``` +gpg2 --gen-key +``` + - ``` +cp ~/.gnupg/secring.gpg ~/.sbt/gpg/secring.asc +``` - publish kafka to local ivy repository - -- ``` + - ``` ./sbt "++2.9.2" clean package publish-local ``` - - publish kafka to local maven repo -- ``` -mvn install:install-file -Dfile=$HOME/.ivy2/local/org.apache/kafka_2.9.2/0.8.0-SNAPSHOT/jars/kafka_2.9.2.jar -DpomFile=$HOME/.ivy2/local/org.apache/kafka_2.9.2/0.8.0-SNAPSHOT/poms/kafka_2.9.2.pom + - ``` +mvn install:install-file -Dfile=$HOME/.ivy2/local/org.apache.kafka/kafka_2.9.2/0.8.0-beta1/jars/kafka_2.9.2.jar -DpomFile=$HOME/.ivy2/local/org.apache.kafka/kafka_2.9.2/0.8.0-beta1/poms/kafka_2.9.2.pom ``` diff --git a/pom.xml b/pom.xml index 85c39faaef..bde18ce0a5 100644 --- a/pom.xml +++ b/pom.xml @@ -130,9 +130,9 @@ test - org.apache + org.apache.kafka kafka_2.9.2 - 0.8.0-SNAPSHOT + 0.8.0-beta1 org.apache.zookeeper From 20a1167dcded98fb53bdfceb79165c6e2d296e7f Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Wed, 31 Jul 2013 22:08:40 +0100 Subject: [PATCH 189/349] fixed formatting --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index d37fb7f222..f6ed071be3 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ cp ~/.gnupg/secring.gpg ~/.sbt/gpg/secring.asc ./sbt "++2.9.2" clean package publish-local ``` - publish kafka to local maven repo - - ``` mvn install:install-file -Dfile=$HOME/.ivy2/local/org.apache.kafka/kafka_2.9.2/0.8.0-beta1/jars/kafka_2.9.2.jar -DpomFile=$HOME/.ivy2/local/org.apache.kafka/kafka_2.9.2/0.8.0-beta1/poms/kafka_2.9.2.pom ``` From 90a82604f8a40fb33e05f5b4d2003198d4263555 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Wed, 31 Jul 2013 22:10:22 +0100 Subject: [PATCH 190/349] fixed formatting --- README.md | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index f6ed071be3..034bfa9e99 100644 --- a/README.md +++ b/README.md @@ -10,18 +10,10 @@ Unfortunately the current beta pom is slightly broken ([KAFKA-974](https://issue - checkout kafka 0.8 branch - Check [KAFKA-939](https://issues.apache.org/jira/browse/KAFKA-939) for instructions on how to handle javadoc erros when publishing - GPG: - - ``` -gpg2 --gen-key -``` - - ``` -cp ~/.gnupg/secring.gpg ~/.sbt/gpg/secring.asc -``` + - ```gpg2 --gen-key``` + - ```cp ~/.gnupg/secring.gpg ~/.sbt/gpg/secring.asc``` - publish kafka to local ivy repository - - ``` -./sbt "++2.9.2" clean package publish-local -``` + - ```./sbt "++2.9.2" clean package publish-local``` - publish kafka to local maven repo - - ``` -mvn install:install-file -Dfile=$HOME/.ivy2/local/org.apache.kafka/kafka_2.9.2/0.8.0-beta1/jars/kafka_2.9.2.jar -DpomFile=$HOME/.ivy2/local/org.apache.kafka/kafka_2.9.2/0.8.0-beta1/poms/kafka_2.9.2.pom -``` + - ```mvn install:install-file -Dfile=$HOME/.ivy2/local/org.apache.kafka/kafka_2.9.2/0.8.0-beta1/jars/kafka_2.9.2.jar -DpomFile=$HOME/.ivy2/local/org.apache.kafka/kafka_2.9.2/0.8.0-beta1/poms/kafka_2.9.2.pom``` From 28789106f0b3e6f3b5ef5767df2d6a0ca6ff02cb Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Wed, 31 Jul 2013 22:34:05 +0100 Subject: [PATCH 191/349] updated to storm 0.9.0-wip21 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bde18ce0a5..0d1641c8a7 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,7 @@ storm storm - 0.9.0-wip15 + 0.9.0-wip21 provided From 3d058660c94d97b15e9700b615a82051b599be34 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sat, 17 Aug 2013 10:07:48 +0100 Subject: [PATCH 192/349] added license --- LICENSE | 191 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..37ec93a14f --- /dev/null +++ b/LICENSE @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From d3eda036ed839d344415dbaab471fef4472a8d9b Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sat, 17 Aug 2013 12:37:43 +0100 Subject: [PATCH 193/349] fixed infinite loop in KafkaSpout --- src/jvm/storm/kafka/PartitionManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 9186953c6a..b9cfd97227 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -151,7 +151,7 @@ private void fill() { for(MessageAndOffset msg: msgs) { _pending.add(_emittedToOffset); _waitingToEmit.add(new MessageAndRealOffset(msg.message(), _emittedToOffset)); - _emittedToOffset = msg.offset(); + _emittedToOffset = msg.nextOffset(); } if(numMessages>0) { LOG.info("Added " + numMessages + " messages from Kafka: " + _consumer.host() + ":" + _partition.partition + " to internal buffers"); From 572a57702dfbe8918b74247a71f5922ebbde7901 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 18 Aug 2013 08:38:58 +0100 Subject: [PATCH 194/349] renamed GlobalPartitionId --- .../kafka/DynamicPartitionConnections.java | 14 ++-- src/jvm/storm/kafka/GlobalPartitionId.java | 39 ----------- src/jvm/storm/kafka/KafkaSpout.java | 2 +- src/jvm/storm/kafka/Partition.java | 48 ++++++++++++++ src/jvm/storm/kafka/PartitionCoordinator.java | 2 +- src/jvm/storm/kafka/PartitionManager.java | 14 ++-- src/jvm/storm/kafka/StaticCoordinator.java | 12 ++-- src/jvm/storm/kafka/ZkCoordinator.java | 22 +++---- .../trident/GlobalPartitionInformation.java | 20 +++--- src/jvm/storm/kafka/trident/KafkaUtils.java | 14 ++-- .../trident/OpaqueTridentKafkaSpout.java | 6 +- .../TransactionalTridentKafkaSpout.java | 4 +- .../kafka/trident/TridentKafkaEmitter.java | 66 ++++++++++++------- 13 files changed, 145 insertions(+), 118 deletions(-) delete mode 100644 src/jvm/storm/kafka/GlobalPartitionId.java create mode 100644 src/jvm/storm/kafka/Partition.java diff --git a/src/jvm/storm/kafka/DynamicPartitionConnections.java b/src/jvm/storm/kafka/DynamicPartitionConnections.java index a5dd1cb332..409ec3717a 100644 --- a/src/jvm/storm/kafka/DynamicPartitionConnections.java +++ b/src/jvm/storm/kafka/DynamicPartitionConnections.java @@ -33,9 +33,9 @@ public DynamicPartitionConnections(KafkaConfig config, IBrokerReader brokerReade _reader = brokerReader; } - public SimpleConsumer register(GlobalPartitionId id) { - HostPort hostPort = _reader.getCurrentBrokers().getHostFor(id.partition); - return register(hostPort, id.partition); + public SimpleConsumer register(Partition partition) { + HostPort hostPort = _reader.getCurrentBrokers().getHostFor(partition.partition); + return register(hostPort, partition.partition); } public SimpleConsumer register(HostPort host, int partition) { @@ -47,8 +47,8 @@ public SimpleConsumer register(HostPort host, int partition) { return info.consumer; } - public SimpleConsumer getConnection(GlobalPartitionId id) { - ConnectionInfo info = _connections.get(id.host); + public SimpleConsumer getConnection(Partition partition) { + ConnectionInfo info = _connections.get(partition.host); if(info != null) return info.consumer; return null; } @@ -62,8 +62,8 @@ public void unregister(HostPort port, int partition) { } } - public void unregister(GlobalPartitionId id) { - unregister(id.host, id.partition); + public void unregister(Partition partition) { + unregister(partition.host, partition.partition); } public void clear() { diff --git a/src/jvm/storm/kafka/GlobalPartitionId.java b/src/jvm/storm/kafka/GlobalPartitionId.java deleted file mode 100644 index 2024a1bbf3..0000000000 --- a/src/jvm/storm/kafka/GlobalPartitionId.java +++ /dev/null @@ -1,39 +0,0 @@ -package storm.kafka; - -import storm.trident.spout.ISpoutPartition; - - -public class GlobalPartitionId implements ISpoutPartition { - public HostPort host; - public int partition; - - public GlobalPartitionId(HostPort host, int partition) { - this.host = host; - this.partition = partition; - } - - @Override - public boolean equals(Object o) { - GlobalPartitionId other = (GlobalPartitionId) o; - return host.equals(other.host) && partition == other.partition; - } - - @Override - public int hashCode() { - return 13 * host.hashCode() + partition; - } - - @Override - public String toString() { - return "GlobalPartitionId{" + - "host=" + host + - ", partition=" + partition + - '}'; - } - - @Override - public String getId() { - return toString(); - } - -} diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index dde2026e81..781e6ceb09 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -79,7 +79,7 @@ public void open(Map conf, final TopologyContext context, final SpoutOutputColle @Override public Object getValueAndReset() { List pms = _coordinator.getMyManagedPartitions(); - Set latestPartitions = new HashSet(); + Set latestPartitions = new HashSet(); for(PartitionManager pm : pms) { latestPartitions.add(pm.getPartition()); } _kafkaOffsetMetric.refreshPartitions(latestPartitions); for(PartitionManager pm : pms) { diff --git a/src/jvm/storm/kafka/Partition.java b/src/jvm/storm/kafka/Partition.java new file mode 100644 index 0000000000..c468925b41 --- /dev/null +++ b/src/jvm/storm/kafka/Partition.java @@ -0,0 +1,48 @@ +package storm.kafka; + +import storm.trident.spout.ISpoutPartition; + +import java.util.Objects; + + +public class Partition implements ISpoutPartition { + + public final HostPort host; + public final int partition; + + public Partition(HostPort host, int partition) { + this.host = host; + this.partition = partition; + } + + @Override + public int hashCode() { + return Objects.hash(host, partition); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final Partition other = (Partition) obj; + return Objects.equals(this.host, other.host) && Objects.equals(this.partition, other.partition); + } + + @Override + public String toString() { + return "Partition{" + + "host=" + host + + ", partition=" + partition + + '}'; + } + + @Override + public String getId() { + return toString(); + } + +} diff --git a/src/jvm/storm/kafka/PartitionCoordinator.java b/src/jvm/storm/kafka/PartitionCoordinator.java index 21f71cd8d7..2ee2009e55 100644 --- a/src/jvm/storm/kafka/PartitionCoordinator.java +++ b/src/jvm/storm/kafka/PartitionCoordinator.java @@ -4,5 +4,5 @@ public interface PartitionCoordinator { List getMyManagedPartitions(); - PartitionManager getManager(GlobalPartitionId id); + PartitionManager getManager(Partition partition); } diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index b9cfd97227..6596f9674b 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -30,10 +30,10 @@ public class PartitionManager { private final CountMetric _fetchAPIMessageCount; static class KafkaMessageId { - public GlobalPartitionId partition; + public Partition partition; public long offset; - public KafkaMessageId(GlobalPartitionId partition, long offset) { + public KafkaMessageId(Partition partition, long offset) { this.partition = partition; this.offset = offset; } @@ -43,7 +43,7 @@ public KafkaMessageId(GlobalPartitionId partition, long offset) { SortedSet _pending = new TreeSet(); Long _committedTo; LinkedList _waitingToEmit = new LinkedList(); - GlobalPartitionId _partition; + Partition _partition; SpoutConfig _spoutConfig; String _topologyInstanceId; SimpleConsumer _consumer; @@ -52,7 +52,7 @@ public KafkaMessageId(GlobalPartitionId partition, long offset) { Map _stormConf; - public PartitionManager(DynamicPartitionConnections connections, String topologyInstanceId, ZkState state, Map stormConf, SpoutConfig spoutConfig, GlobalPartitionId id) { + public PartitionManager(DynamicPartitionConnections connections, String topologyInstanceId, ZkState state, Map stormConf, SpoutConfig spoutConfig, Partition id) { _partition = id; _connections = connections; _spoutConfig = spoutConfig; @@ -192,11 +192,11 @@ public void commit() { Map data = (Map)ImmutableMap.builder() .put("topology", ImmutableMap.of("id", _topologyInstanceId, - "name", _stormConf.get(Config.TOPOLOGY_NAME))) + "name", _stormConf.get(Config.TOPOLOGY_NAME))) .put("offset", committedTo) .put("partition", _partition.partition) .put("broker", ImmutableMap.of("host", _partition.host.host, - "port", _partition.host.port)) + "port", _partition.host.port)) .put("topic", _spoutConfig.topic).build(); _state.writeJSON(committedPath(), data); @@ -227,7 +227,7 @@ public long lastCompletedOffset() { } } - public GlobalPartitionId getPartition() { + public Partition getPartition() { return _partition; } diff --git a/src/jvm/storm/kafka/StaticCoordinator.java b/src/jvm/storm/kafka/StaticCoordinator.java index d3215cb89a..6f97c8b727 100644 --- a/src/jvm/storm/kafka/StaticCoordinator.java +++ b/src/jvm/storm/kafka/StaticCoordinator.java @@ -7,14 +7,14 @@ public class StaticCoordinator implements PartitionCoordinator { - Map _managers = new HashMap(); + Map _managers = new HashMap(); List _allManagers = new ArrayList(); public StaticCoordinator(DynamicPartitionConnections connections, Map stormConf, SpoutConfig config, ZkState state, int taskIndex, int totalTasks, String topologyInstanceId) { StaticHosts hosts = (StaticHosts) config.hosts; - List allPartitionIds = hosts.getPartitionInformation().getOrderedPartitions(); - for(int i=taskIndex; i partitions = hosts.getPartitionInformation().getOrderedPartitions(); + for(int i=taskIndex; i getMyManagedPartitions() { return _allManagers; } - public PartitionManager getManager(GlobalPartitionId id) { - return _managers.get(id); + public PartitionManager getManager(Partition partition) { + return _managers.get(partition); } } diff --git a/src/jvm/storm/kafka/ZkCoordinator.java b/src/jvm/storm/kafka/ZkCoordinator.java index d94c5fd585..3fef88af41 100644 --- a/src/jvm/storm/kafka/ZkCoordinator.java +++ b/src/jvm/storm/kafka/ZkCoordinator.java @@ -14,7 +14,7 @@ public class ZkCoordinator implements PartitionCoordinator { int _taskIndex; int _totalTasks; String _topologyInstanceId; - Map _managers = new HashMap(); + Map _managers = new HashMap(); List _cachedList; Long _lastRefreshTime = null; int _refreshFreqMs; @@ -52,29 +52,29 @@ void refresh() { try { LOG.info("Refreshing partition manager connections"); GlobalPartitionInformation brokerInfo = _reader.getBrokerInfo(); - Set mine = new HashSet(); - for (GlobalPartitionId partitionId: brokerInfo){ + Set mine = new HashSet(); + for (Partition partitionId: brokerInfo){ if(myOwnership(partitionId)) { mine.add(partitionId); } } - Set curr = _managers.keySet(); - Set newPartitions = new HashSet(mine); + Set curr = _managers.keySet(); + Set newPartitions = new HashSet(mine); newPartitions.removeAll(curr); - Set deletedPartitions = new HashSet(curr); + Set deletedPartitions = new HashSet(curr); deletedPartitions.removeAll(mine); LOG.info("Deleted partition managers: " + deletedPartitions.toString()); - for(GlobalPartitionId id: deletedPartitions) { + for(Partition id: deletedPartitions) { PartitionManager man = _managers.remove(id); man.close(); } LOG.info("New partition managers: " + newPartitions.toString()); - for(GlobalPartitionId id: newPartitions) { + for(Partition id: newPartitions) { PartitionManager man = new PartitionManager(_connections, _topologyInstanceId, _state, _stormConf, _spoutConfig, id); _managers.put(id, man); } @@ -87,11 +87,11 @@ void refresh() { } @Override - public PartitionManager getManager(GlobalPartitionId id) { - return _managers.get(id); + public PartitionManager getManager(Partition partition) { + return _managers.get(partition); } - private boolean myOwnership(GlobalPartitionId id) { + private boolean myOwnership(Partition id) { int val = Math.abs(id.host.hashCode() + 23 * id.partition); return val % _totalTasks == _taskIndex; } diff --git a/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java b/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java index 41df68bdf8..139a0d783d 100644 --- a/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java +++ b/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java @@ -1,6 +1,6 @@ package storm.kafka.trident; -import storm.kafka.GlobalPartitionId; +import storm.kafka.Partition; import storm.kafka.HostPort; import java.io.Serializable; @@ -10,7 +10,7 @@ * Date: 14/05/2013 * Time: 19:18 */ -public class GlobalPartitionInformation implements Iterable, Serializable { +public class GlobalPartitionInformation implements Iterable, Serializable { private Map partitionMap; @@ -33,28 +33,28 @@ public HostPort getHostFor(Integer partitionId) { return partitionMap.get(partitionId); } - public List getOrderedPartitions(){ - List partitionIds = new LinkedList(); + public List getOrderedPartitions(){ + List partitions = new LinkedList(); for (Map.Entry partition : partitionMap.entrySet()) { - partitionIds.add(new GlobalPartitionId(partition.getValue(), partition.getKey())); + partitions.add(new Partition(partition.getValue(), partition.getKey())); } - return partitionIds; + return partitions; } @Override - public Iterator iterator() { + public Iterator iterator() { final Iterator> iterator = partitionMap.entrySet().iterator(); - return new Iterator() { + return new Iterator() { @Override public boolean hasNext() { return iterator.hasNext(); } @Override - public GlobalPartitionId next() { + public Partition next() { Map.Entry next = iterator.next(); - return new GlobalPartitionId(next.getValue(), next.getKey()); + return new Partition(next.getValue(), next.getKey()); } @Override diff --git a/src/jvm/storm/kafka/trident/KafkaUtils.java b/src/jvm/storm/kafka/trident/KafkaUtils.java index 585554cee4..efe4fefdb7 100644 --- a/src/jvm/storm/kafka/trident/KafkaUtils.java +++ b/src/jvm/storm/kafka/trident/KafkaUtils.java @@ -43,8 +43,8 @@ public static long getOffset(SimpleConsumer consumer, String topic, int partitio } public static class KafkaOffsetMetric implements IMetric { - Map _partitionToOffset = new HashMap(); - Set _partitions; + Map _partitionToOffset = new HashMap(); + Set _partitions; String _topic; DynamicPartitionConnections _connections; @@ -53,7 +53,7 @@ public KafkaOffsetMetric(String topic, DynamicPartitionConnections connections) _connections = connections; } - public void setLatestEmittedOffset(GlobalPartitionId partition, long offset) { + public void setLatestEmittedOffset(Partition partition, long offset) { _partitionToOffset.put(partition, offset); } @@ -65,8 +65,8 @@ public Object getValueAndReset() { long totalLatestEmittedOffset = 0; HashMap ret = new HashMap(); if(_partitions != null && _partitions.size() == _partitionToOffset.size()) { - for(Map.Entry e : _partitionToOffset.entrySet()) { - GlobalPartitionId partition = e.getKey(); + for(Map.Entry e : _partitionToOffset.entrySet()) { + Partition partition = e.getKey(); SimpleConsumer consumer = _connections.getConnection(partition); if(consumer == null) { LOG.warn("partitionToOffset contains partition not found in _connections. Stale partition data?"); @@ -99,9 +99,9 @@ public Object getValueAndReset() { return null; } - public void refreshPartitions(Set partitions) { + public void refreshPartitions(Set partitions) { _partitions = partitions; - Iterator it = _partitionToOffset.keySet().iterator(); + Iterator it = _partitionToOffset.keySet().iterator(); while(it.hasNext()) { if(!partitions.contains(it.next())) it.remove(); } diff --git a/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java b/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java index 087c7b9fe8..0f6e6c518b 100644 --- a/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java +++ b/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java @@ -2,14 +2,14 @@ import backtype.storm.task.TopologyContext; import backtype.storm.tuple.Fields; -import storm.kafka.GlobalPartitionId; +import storm.kafka.Partition; import storm.trident.spout.IOpaquePartitionedTridentSpout; import java.util.Map; import java.util.UUID; -public class OpaqueTridentKafkaSpout implements IOpaquePartitionedTridentSpout { +public class OpaqueTridentKafkaSpout implements IOpaquePartitionedTridentSpout { TridentKafkaConfig _config; @@ -20,7 +20,7 @@ public OpaqueTridentKafkaSpout(TridentKafkaConfig config) { } @Override - public IOpaquePartitionedTridentSpout.Emitter getEmitter(Map conf, TopologyContext context) { + public IOpaquePartitionedTridentSpout.Emitter getEmitter(Map conf, TopologyContext context) { return new TridentKafkaEmitter(conf, context, _config, _topologyInstanceId).asOpaqueEmitter(); } diff --git a/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java b/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java index efe8075eed..173a98f575 100644 --- a/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java +++ b/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java @@ -2,14 +2,14 @@ import backtype.storm.task.TopologyContext; import backtype.storm.tuple.Fields; -import storm.kafka.GlobalPartitionId; +import storm.kafka.Partition; import storm.trident.spout.IPartitionedTridentSpout; import java.util.Map; import java.util.UUID; -public class TransactionalTridentKafkaSpout implements IPartitionedTridentSpout { +public class TransactionalTridentKafkaSpout implements IPartitionedTridentSpout { TridentKafkaConfig _config; String _topologyInstanceId = UUID.randomUUID().toString(); diff --git a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java index 5ef88d2ded..282e67a1a1 100644 --- a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java +++ b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java @@ -16,7 +16,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import storm.kafka.DynamicPartitionConnections; -import storm.kafka.GlobalPartitionId; +import storm.kafka.Partition; import storm.trident.operation.TridentCollector; import storm.trident.spout.IOpaquePartitionedTridentSpout; import storm.trident.spout.IPartitionedTridentSpout; @@ -56,14 +56,14 @@ public TridentKafkaEmitter(Map conf, TopologyContext context, TridentKafkaConfig } - private Map failFastEmitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map lastMeta) { + private Map failFastEmitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map lastMeta) { SimpleConsumer consumer = _connections.register(partition); Map ret = doEmitNewPartitionBatch(consumer, partition, collector, lastMeta); _kafkaOffsetMetric.setLatestEmittedOffset(partition, (Long) ret.get("offset")); return ret; } - private Map emitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map lastMeta) { + private Map emitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map lastMeta) { try { return failFastEmitNewPartitionBatch(attempt, collector, partition, lastMeta); } catch (FailedFetchException e) { @@ -83,7 +83,7 @@ private Map emitNewPartitionBatch(TransactionAttempt attempt, TridentCollector c } } - private Map doEmitNewPartitionBatch(SimpleConsumer consumer, GlobalPartitionId partition, TridentCollector collector, Map lastMeta) { + private Map doEmitNewPartitionBatch(SimpleConsumer consumer, Partition partition, TridentCollector collector, Map lastMeta) { long offset; if (lastMeta != null) { String lastInstanceId = null; @@ -127,7 +127,7 @@ private Map doEmitNewPartitionBatch(SimpleConsumer consumer, GlobalPartitionId p return newMeta; } - private ByteBufferMessageSet fetchMessages(SimpleConsumer consumer, GlobalPartitionId partition, long offset) { + private ByteBufferMessageSet fetchMessages(SimpleConsumer consumer, Partition partition, long offset) { ByteBufferMessageSet msgs; long start = System.nanoTime(); FetchRequestBuilder builder = new FetchRequestBuilder(); @@ -148,7 +148,7 @@ private ByteBufferMessageSet fetchMessages(SimpleConsumer consumer, GlobalPartit * @param partition * @param meta */ - private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, GlobalPartitionId partition, Map meta) { + private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map meta) { LOG.info("re-emitting batch, attempt " + attempt); String instanceId = (String) meta.get("instanceId"); if (!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) { @@ -181,32 +181,38 @@ private void clear() { _connections.clear(); } - private List orderPartitions(GlobalPartitionInformation partitions) { + private List orderPartitions(GlobalPartitionInformation partitions) { return partitions.getOrderedPartitions(); } - private void refresh(List list) { + private void refresh(List list) { _connections.clear(); - _kafkaOffsetMetric.refreshPartitions(new HashSet(list)); + _kafkaOffsetMetric.refreshPartitions(new HashSet(list)); } - public IOpaquePartitionedTridentSpout.Emitter asOpaqueEmitter() { + public IOpaquePartitionedTridentSpout.Emitter asOpaqueEmitter() { - return new IOpaquePartitionedTridentSpout.Emitter() { + return new IOpaquePartitionedTridentSpout.Emitter() { + /** + * Emit a batch of tuples for a partition/transaction. + * + * Return the metadata describing this batch that will be used as lastPartitionMeta + * for defining the parameters of the next batch. + */ @Override - public Map emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, GlobalPartitionId globalPartitionId, Map map) { - return emitNewPartitionBatch(transactionAttempt, tridentCollector, globalPartitionId, map); + public Map emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, Partition partition, Map map) { + return emitNewPartitionBatch(transactionAttempt, tridentCollector, partition, map); } @Override - public void refreshPartitions(List globalPartitionIds) { - refresh(globalPartitionIds); + public void refreshPartitions(List partitions) { + refresh(partitions); } @Override - public List getOrderedPartitions(GlobalPartitionInformation partitionInformation) { + public List getOrderedPartitions(GlobalPartitionInformation partitionInformation) { return orderPartitions(partitionInformation); } @@ -218,25 +224,37 @@ public void close() { } public IPartitionedTridentSpout.Emitter asTransactionalEmitter() { - return new IPartitionedTridentSpout.Emitter() { + return new IPartitionedTridentSpout.Emitter() { + /** + * Emit a batch of tuples for a partition/transaction that's never been emitted before. + * Return the metadata that can be used to reconstruct this partition/batch in the future. + */ @Override - public Map emitPartitionBatchNew(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, GlobalPartitionId globalPartitionId, Map map) { - return failFastEmitNewPartitionBatch(transactionAttempt, tridentCollector, globalPartitionId, map); + public Map emitPartitionBatchNew(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, Partition partition, Map map) { + return failFastEmitNewPartitionBatch(transactionAttempt, tridentCollector, partition, map); } + /** + * Emit a batch of tuples for a partition/transaction that has been emitted before, using + * the metadata created when it was first emitted. + */ @Override - public void emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, GlobalPartitionId globalPartitionId, Map map) { - reEmitPartitionBatch(transactionAttempt, tridentCollector, globalPartitionId, map); + public void emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, Partition partition, Map map) { + reEmitPartitionBatch(transactionAttempt, tridentCollector, partition, map); } + /** + * This method is called when this task is responsible for a new set of partitions. Should be used + * to manage things like connections to brokers. + */ @Override - public void refreshPartitions(List globalPartitionIds) { - refresh(globalPartitionIds); + public void refreshPartitions(List partitions) { + refresh(partitions); } @Override - public List getOrderedPartitions(GlobalPartitionInformation partitionInformation) { + public List getOrderedPartitions(GlobalPartitionInformation partitionInformation) { return orderPartitions(partitionInformation); } From d0d7acb26aee4b1f691ea3c8bdf4cc9e220f693f Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Wed, 21 Aug 2013 22:11:30 +0100 Subject: [PATCH 195/349] removed JDK 7 dependency --- src/jvm/storm/kafka/Partition.java | 7 +++---- src/jvm/storm/kafka/ZkCoordinator.java | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/jvm/storm/kafka/Partition.java b/src/jvm/storm/kafka/Partition.java index c468925b41..6de0f6a1fd 100644 --- a/src/jvm/storm/kafka/Partition.java +++ b/src/jvm/storm/kafka/Partition.java @@ -1,9 +1,8 @@ package storm.kafka; +import com.google.common.base.Objects; import storm.trident.spout.ISpoutPartition; -import java.util.Objects; - public class Partition implements ISpoutPartition { @@ -17,7 +16,7 @@ public Partition(HostPort host, int partition) { @Override public int hashCode() { - return Objects.hash(host, partition); + return Objects.hashCode(host, partition); } @Override @@ -29,7 +28,7 @@ public boolean equals(Object obj) { return false; } final Partition other = (Partition) obj; - return Objects.equals(this.host, other.host) && Objects.equals(this.partition, other.partition); + return Objects.equal(this.host, other.host) && Objects.equal(this.partition, other.partition); } @Override diff --git a/src/jvm/storm/kafka/ZkCoordinator.java b/src/jvm/storm/kafka/ZkCoordinator.java index 3fef88af41..d457bddbaf 100644 --- a/src/jvm/storm/kafka/ZkCoordinator.java +++ b/src/jvm/storm/kafka/ZkCoordinator.java @@ -8,7 +8,7 @@ import java.util.*; public class ZkCoordinator implements PartitionCoordinator { - public static Logger LOG = LoggerFactory.getLogger(ZkCoordinator.class); + public static final Logger LOG = LoggerFactory.getLogger(ZkCoordinator.class); SpoutConfig _spoutConfig; int _taskIndex; From e7b4c09b6859e6b866d8b45b948c7d69b1cc0ceb Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Tue, 27 Aug 2013 21:12:51 +0200 Subject: [PATCH 196/349] ISSUE-36: equals() should compare against input parameter --- src/jvm/storm/starter/tools/RankableObjectWithFields.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jvm/storm/starter/tools/RankableObjectWithFields.java b/src/jvm/storm/starter/tools/RankableObjectWithFields.java index 43be75a77f..4d571f2625 100644 --- a/src/jvm/storm/starter/tools/RankableObjectWithFields.java +++ b/src/jvm/storm/starter/tools/RankableObjectWithFields.java @@ -82,7 +82,7 @@ else if (delta < 0) { @Override public boolean equals(Object o) { - if (this == obj) { + if (this == o) { return true; } if (!(o instanceof RankableObjectWithFields)) { From e3450917b100ee46ad5ff334e815fbb015686ccb Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Tue, 27 Aug 2013 21:18:19 +0200 Subject: [PATCH 197/349] storm-starter actually uses leiningen 2.0 now --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 3191b2ee64..d760516be1 100644 --- a/README.markdown +++ b/README.markdown @@ -10,7 +10,7 @@ More information about Storm can be found on the [project page](http://github.co ## Running an example with Leiningen -Install Leiningen by following the installation instructions [here](https://github.com/technomancy/leiningen). The storm-starter build uses Leiningen 1.7.1. +Install Leiningen by following the installation instructions [here](https://github.com/technomancy/leiningen). The storm-starter build uses Leiningen 2.0. ### To run a Java example: From 0b7acda1df88301c77f212673789a55d3c445ad2 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 11:17:09 +0200 Subject: [PATCH 198/349] RankableObjectWithFields implements Serializable to fix NotSerializableException errors --- src/jvm/storm/starter/tools/RankableObjectWithFields.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/jvm/storm/starter/tools/RankableObjectWithFields.java b/src/jvm/storm/starter/tools/RankableObjectWithFields.java index 4d571f2625..38db6e9231 100644 --- a/src/jvm/storm/starter/tools/RankableObjectWithFields.java +++ b/src/jvm/storm/starter/tools/RankableObjectWithFields.java @@ -1,5 +1,6 @@ package storm.starter.tools; +import java.io.Serializable; import java.util.List; import backtype.storm.tuple.Tuple; @@ -13,8 +14,9 @@ * This class can be used, for instance, to track the number of occurrences of an object in a Storm topology. * */ -public class RankableObjectWithFields implements Rankable { +public class RankableObjectWithFields implements Rankable, Serializable { + private static final long serialVersionUID = -9102878650001058090L; private static final String toStringSeparator = "|"; private final Object obj; From f26f5974437c32510bde0fdf5dcdc053b42be1f6 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 11:23:08 +0200 Subject: [PATCH 199/349] Explain how to run unit tests with Maven --- README.markdown | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.markdown b/README.markdown index d760516be1..9bb623fb32 100644 --- a/README.markdown +++ b/README.markdown @@ -43,3 +43,11 @@ mvn -f m2-pom.xml package ``` This will package your code and all the non-Storm dependencies into a single "uberjar" at the path `target/storm-starter-{version}-jar-with-dependencies.jar`. + + +### To run unit tests + +Use the following Maven command to run the unit tests that ship with storm-starter. Unfortunately `lein test` does not +yet run the included unit tests. + + $ mvn -f m2-pom.xml test From ab44424d85054e6873d94d2094c2dacfdbbac9b6 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 11:23:54 +0200 Subject: [PATCH 200/349] README: Use 120 chars max per line --- README.markdown | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.markdown b/README.markdown index 9bb623fb32..0379056875 100644 --- a/README.markdown +++ b/README.markdown @@ -1,6 +1,7 @@ # Example Storm topologies -storm-starter contains a variety of examples of using Storm. If this is your first time checking out Storm, check out these topologies first: +storm-starter contains a variety of examples of using Storm. If this is your first time checking out Storm, check out +these topologies first: 1. ExclamationTopology: Basic topology written in all Java 2. WordCountTopology: Basic topology that makes use of multilang by implementing one bolt in Python @@ -10,7 +11,8 @@ More information about Storm can be found on the [project page](http://github.co ## Running an example with Leiningen -Install Leiningen by following the installation instructions [here](https://github.com/technomancy/leiningen). The storm-starter build uses Leiningen 2.0. +Install Leiningen by following the installation instructions [here](https://github.com/technomancy/leiningen). The +storm-starter build uses Leiningen 2.0. ### To run a Java example: @@ -30,7 +32,8 @@ lein run -m storm.starter.clj.word-count ## Maven -Maven is an alternative to Leiningen. storm-starter contains m2-pom.xml which can be used with Maven using the -f option. For example, to compile and run `WordCountTopology` in local mode, use this command: +Maven is an alternative to Leiningen. storm-starter contains m2-pom.xml which can be used with Maven using the -f +option. For example, to compile and run `WordCountTopology` in local mode, use this command: ``` mvn -f m2-pom.xml compile exec:java -Dexec.classpathScope=compile -Dexec.mainClass=storm.starter.WordCountTopology @@ -42,7 +45,8 @@ You can package a jar suitable for submitting to a cluster with this command: mvn -f m2-pom.xml package ``` -This will package your code and all the non-Storm dependencies into a single "uberjar" at the path `target/storm-starter-{version}-jar-with-dependencies.jar`. +This will package your code and all the non-Storm dependencies into a single "uberjar" at the path +`target/storm-starter-{version}-jar-with-dependencies.jar`. ### To run unit tests From 84b2eba70a0c9c6d5a790c0686d0e835bbad8641 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 11:25:57 +0200 Subject: [PATCH 201/349] Reformat README --- README.markdown | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/README.markdown b/README.markdown index 0379056875..5c21872011 100644 --- a/README.markdown +++ b/README.markdown @@ -9,41 +9,38 @@ these topologies first: More information about Storm can be found on the [project page](http://github.com/nathanmarz/storm). + ## Running an example with Leiningen -Install Leiningen by following the installation instructions [here](https://github.com/technomancy/leiningen). The -storm-starter build uses Leiningen 2.0. +Install Leiningen by following the [leiningen installation instructions](https://github.com/technomancy/leiningen). +The storm-starter build uses Leiningen 2.0. + ### To run a Java example: -``` -lein deps -lein compile -java -cp $(lein classpath) storm.starter.ExclamationTopology -``` + $ lein deps + $ lein compile + $ java -cp $(lein classpath) storm.starter.ExclamationTopology + ### To run a Clojure example: -``` -lein deps -lein compile -lein run -m storm.starter.clj.word-count -``` + $ lein deps + $ lein compile + $ lein run -m storm.starter.clj.word-count + ## Maven -Maven is an alternative to Leiningen. storm-starter contains m2-pom.xml which can be used with Maven using the -f +Maven is an alternative to Leiningen. storm-starter contains m2-pom.xml which can be used with Maven using the `-f` option. For example, to compile and run `WordCountTopology` in local mode, use this command: -``` -mvn -f m2-pom.xml compile exec:java -Dexec.classpathScope=compile -Dexec.mainClass=storm.starter.WordCountTopology -``` + + $ mvn -f m2-pom.xml compile exec:java -Dexec.classpathScope=compile -Dexec.mainClass=storm.starter.WordCountTopology You can package a jar suitable for submitting to a cluster with this command: -``` -mvn -f m2-pom.xml package -``` + $ mvn -f m2-pom.xml package This will package your code and all the non-Storm dependencies into a single "uberjar" at the path `target/storm-starter-{version}-jar-with-dependencies.jar`. From 84cb0355e6827f967784d02ce85f1ba373932d4c Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 11:28:25 +0200 Subject: [PATCH 202/349] Update README --- README.markdown | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.markdown b/README.markdown index 5c21872011..8b44fde529 100644 --- a/README.markdown +++ b/README.markdown @@ -1,13 +1,14 @@ # Example Storm topologies -storm-starter contains a variety of examples of using Storm. If this is your first time checking out Storm, check out +storm-starter contains a variety of examples of using Storm. If this is your first time working with Storm, check out these topologies first: -1. ExclamationTopology: Basic topology written in all Java -2. WordCountTopology: Basic topology that makes use of multilang by implementing one bolt in Python -3. ReachTopology: Example of complex DRPC on top of Storm +1. [ExclamationTopology](src/jvm/storm/starter/ExclamationTopology.java): Basic topology written in all Java +2. [WordCountTopology](src/jvm/storm/starter/WordCountTopology.java): Basic topology that makes use of multilang by + implementing one bolt in Python +3. [ReachTopology](src/jvm/storm/starter/ReachTopology.java): Example of complex DRPC on top of Storm -More information about Storm can be found on the [project page](http://github.com/nathanmarz/storm). +See the [Storm project page](http://github.com/nathanmarz/storm) for more information. ## Running an example with Leiningen From a9cb8cbfbe751ddda233c67aea26bd7c52dee062 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 11:37:40 +0200 Subject: [PATCH 203/349] Restructure README and add more docs --- README.markdown | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/README.markdown b/README.markdown index 8b44fde529..1b6099ee2d 100644 --- a/README.markdown +++ b/README.markdown @@ -1,4 +1,4 @@ -# Example Storm topologies +# storm-starter: Example Storm Topologies storm-starter contains a variety of examples of using Storm. If this is your first time working with Storm, check out these topologies first: @@ -11,35 +11,49 @@ these topologies first: See the [Storm project page](http://github.com/nathanmarz/storm) for more information. -## Running an example with Leiningen +# Using storm-starter with Leiningen -Install Leiningen by following the [leiningen installation instructions](https://github.com/technomancy/leiningen). -The storm-starter build uses Leiningen 2.0. +## Install Leiningen +The storm-starter build uses [Leiningen](http://leiningen.org/) 2.0. Install Leiningen by following the +[leiningen installation instructions](https://github.com/technomancy/leiningen). -### To run a Java example: + +## Running topologies with Leiningen + +### To run a Java topology $ lein deps $ lein compile $ java -cp $(lein classpath) storm.starter.ExclamationTopology -### To run a Clojure example: +### To run a Clojure topology: $ lein deps $ lein compile $ lein run -m storm.starter.clj.word-count -## Maven +# Using storm-starter with Maven + +## Install Maven + +[Maven](http://maven.apache.org/) is an alternative to Leiningen. Install Maven (preferably version 3.x) by following +the [Maven installation instructions](http://maven.apache.org/download.cgi). -Maven is an alternative to Leiningen. storm-starter contains m2-pom.xml which can be used with Maven using the `-f` -option. For example, to compile and run `WordCountTopology` in local mode, use this command: +## Running topologies with Maven + +storm-starter contains [m2-pom.xml](m2-pom.xml) which can be used with Maven using the `-f` option. For example, to +compile and run `WordCountTopology` in local mode, use the command: $ mvn -f m2-pom.xml compile exec:java -Dexec.classpathScope=compile -Dexec.mainClass=storm.starter.WordCountTopology -You can package a jar suitable for submitting to a cluster with this command: + +## Packaging storm-starter for use on a Storm cluster + +You can package a jar suitable for submitting to a Storm cluster with the command: $ mvn -f m2-pom.xml package @@ -47,7 +61,7 @@ This will package your code and all the non-Storm dependencies into a single "ub `target/storm-starter-{version}-jar-with-dependencies.jar`. -### To run unit tests +## Running unit tests Use the following Maven command to run the unit tests that ship with storm-starter. Unfortunately `lein test` does not yet run the included unit tests. From cfaf58a4512cc80dde84f7c15e783ebbe5e584f2 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 11:38:47 +0200 Subject: [PATCH 204/349] Update main heading --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 1b6099ee2d..8dee7acaf3 100644 --- a/README.markdown +++ b/README.markdown @@ -1,4 +1,4 @@ -# storm-starter: Example Storm Topologies +# Example Storm Topologies storm-starter contains a variety of examples of using Storm. If this is your first time working with Storm, check out these topologies first: From 5a92564e0f47cc411117c892e51866b5547f1266 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 11:41:27 +0200 Subject: [PATCH 205/349] Add table of contents --- README.markdown | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 8dee7acaf3..ef35c624e2 100644 --- a/README.markdown +++ b/README.markdown @@ -1,5 +1,22 @@ # Example Storm Topologies +Learn to use Storm! + +--- + +Table of Contents + +* Getting started +* Using storm-starter with Leiningen +* Using storm-starter with Maven + +--- + + + + +# Getting started + storm-starter contains a variety of examples of using Storm. If this is your first time working with Storm, check out these topologies first: @@ -8,9 +25,11 @@ these topologies first: implementing one bolt in Python 3. [ReachTopology](src/jvm/storm/starter/ReachTopology.java): Example of complex DRPC on top of Storm -See the [Storm project page](http://github.com/nathanmarz/storm) for more information. +See the [Storm project page](http://github.com/nathanmarz/storm) for more information on how Storm works.. + + # Using storm-starter with Leiningen ## Install Leiningen @@ -35,6 +54,8 @@ The storm-starter build uses [Leiningen](http://leiningen.org/) 2.0. Install Le $ lein run -m storm.starter.clj.word-count + + # Using storm-starter with Maven ## Install Maven From f64b52acaa9b758acd1c400b142e5c229585d9b5 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 11:43:52 +0200 Subject: [PATCH 206/349] Update Getting Started section --- README.markdown | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index ef35c624e2..fa4a6501fb 100644 --- a/README.markdown +++ b/README.markdown @@ -25,7 +25,12 @@ these topologies first: implementing one bolt in Python 3. [ReachTopology](src/jvm/storm/starter/ReachTopology.java): Example of complex DRPC on top of Storm -See the [Storm project page](http://github.com/nathanmarz/storm) for more information on how Storm works.. +After you have familiarized yourself with these topologies, take a look at the other topopologies in +[src/jvm/storm/starter/](src/jvm/storm/starter/) such as [RollingTopWords](src/jvm/storm/starter/RollingTopWords.java) +for more advanced implementations. + +If you want to learn more about how Storm works, please head over to the +[Storm project page](http://github.com/nathanmarz/storm). From b9c57eb9ac20cbb44af0c4fd64c8de3f219bf3c3 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 11:57:25 +0200 Subject: [PATCH 207/349] PR-3: Add step-by-step instructions for the curious but unsavvy Most of the original pull request (dated two years ago) was added to the README. See https://github.com/nathanmarz/storm-starter/pull/3 for the original pull request. --- README.markdown | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.markdown b/README.markdown index fa4a6501fb..963e9c2509 100644 --- a/README.markdown +++ b/README.markdown @@ -17,6 +17,20 @@ Table of Contents # Getting started +## Prerequisites + +First, you need `java` and `git` installed and in your user's `PATH`. Also, two of the examples in storm-starter +require Python and Ruby. + +Next, make sure you have the storm-starter code available on your machine. Git/GitHub beginners may want to use the +following command to download the latest storm-starter code and change to the new directory that contains the downloaded +code. + + $ git clone git://github.com/nathanmarz/storm-starter.git && cd storm-starter + + +## storm-starter overview + storm-starter contains a variety of examples of using Storm. If this is your first time working with Storm, check out these topologies first: From 2ace708d42984057f1f29ac3a300ac296b4b7edb Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 12:19:32 +0200 Subject: [PATCH 208/349] AbstractRankerBolt: use log level DEBUG instead INFO so that the bolt is more silent on the logging side --- src/jvm/storm/starter/bolt/AbstractRankerBolt.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jvm/storm/starter/bolt/AbstractRankerBolt.java b/src/jvm/storm/starter/bolt/AbstractRankerBolt.java index 89e26551f6..27d65a0ce3 100644 --- a/src/jvm/storm/starter/bolt/AbstractRankerBolt.java +++ b/src/jvm/storm/starter/bolt/AbstractRankerBolt.java @@ -64,7 +64,7 @@ protected Rankings getRankings() { @Override public final void execute(Tuple tuple, BasicOutputCollector collector) { if (TupleHelpers.isTickTuple(tuple)) { - getLogger().info("Received tick tuple, triggering emit of current rankings"); + getLogger().debug("Received tick tuple, triggering emit of current rankings"); emitRankings(collector); } else { @@ -76,7 +76,7 @@ public final void execute(Tuple tuple, BasicOutputCollector collector) { private void emitRankings(BasicOutputCollector collector) { collector.emit(new Values(rankings)); - getLogger().info("Rankings: " + rankings); + getLogger().debug("Rankings: " + rankings); } @Override From 8e1e3cc1ced2e785c2ce47fd8708dcf2c5a2e8ec Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 12:20:40 +0200 Subject: [PATCH 209/349] RollingCountBolt: use log level DEBUG to tell that a tick tuple was received --- src/jvm/storm/starter/bolt/RollingCountBolt.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jvm/storm/starter/bolt/RollingCountBolt.java b/src/jvm/storm/starter/bolt/RollingCountBolt.java index 848fedb14a..0066b61b8e 100644 --- a/src/jvm/storm/starter/bolt/RollingCountBolt.java +++ b/src/jvm/storm/starter/bolt/RollingCountBolt.java @@ -80,7 +80,7 @@ public void prepare(Map stormConf, TopologyContext context, OutputCollector coll @Override public void execute(Tuple tuple) { if (TupleHelpers.isTickTuple(tuple)) { - LOG.info("Received tick tuple, triggering emit of current window counts"); + LOG.debug("Received tick tuple, triggering emit of current window counts"); emitCurrentWindowCounts(); } else { From 4c714d03035fd6075392bd0df7577c68d993a366 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 12:36:48 +0200 Subject: [PATCH 210/349] Rankings: make updateWith() thread-safe This fixes a ConcurrentModificationException that could be triggered when using log4j to log Rankings#toString() from within a bolt's execute() method. Apparently a separate thread is used by log4j to perform the logging, whose execution path could interweave with the bolt's own thread. Even though this is not necessarily a deficiency of the Rankings class (which does not claim to be thread-safe), we still decided to make Rankings#updateWith() thread-safe so that users do not run into errors when using log4j & Co. for logging purposes. In other words, we decided to add this fix for the convenience of our users. --- m2-pom.xml | 6 +++ src/jvm/storm/starter/tools/Rankings.java | 8 ++-- .../jvm/storm/starter/tools/RankingsTest.java | 42 +++++++++++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/m2-pom.xml b/m2-pom.xml index ba5c760fa5..24293bca1a 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -53,6 +53,12 @@ 2.0M8 test + + org.jmock + jmock + 2.6.0 + test + storm storm diff --git a/src/jvm/storm/starter/tools/Rankings.java b/src/jvm/storm/starter/tools/Rankings.java index 4eee168835..d6fd34ca96 100644 --- a/src/jvm/storm/starter/tools/Rankings.java +++ b/src/jvm/storm/starter/tools/Rankings.java @@ -50,9 +50,11 @@ public void updateWith(Rankings other) { } public void updateWith(Rankable r) { - addOrReplace(r); - rerank(); - shrinkRankingsIfNeeded(); + synchronized(rankedItems) { + addOrReplace(r); + rerank(); + shrinkRankingsIfNeeded(); + } } private void addOrReplace(Rankable r) { diff --git a/test/jvm/storm/starter/tools/RankingsTest.java b/test/jvm/storm/starter/tools/RankingsTest.java index 682b504482..b95a86f85a 100644 --- a/test/jvm/storm/starter/tools/RankingsTest.java +++ b/test/jvm/storm/starter/tools/RankingsTest.java @@ -4,6 +4,9 @@ import java.util.List; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import org.jmock.lib.concurrent.Blitzer; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -203,4 +206,43 @@ public void shouldNotRankDuplicateObjectsMoreThanOnce(List duplicates) // then assertThat(rankings.size()).isEqualTo(1); } + + @Test + public void updatingWithNewRankablesShouldBeThreadSafe() throws InterruptedException { + // given + final List entries = ImmutableList.of(A, B, C, D); + final Rankings rankings = new Rankings(entries.size()); + + // We are capturing exceptions thrown in Blitzer's child threads into this data structure so that we can properly + // pass/fail this test. The reason is that Blitzer doesn't report exceptions, which is a known bug in Blitzer + // (JMOCK-263). See https://github.com/jmock-developers/jmock-library/issues/22 for more information. + final List exceptions = Lists.newArrayList(); + Blitzer blitzer = new Blitzer(1000); + + // when + blitzer.blitz(new Runnable() { + public void run() { + for (Rankable r : entries) { + try { + rankings.updateWith(r); + } + catch (RuntimeException e) { + synchronized(exceptions) { + exceptions.add(e); + } + } + } + } + }); + blitzer.shutdown(); + + // then + // + if (!exceptions.isEmpty()) { + for (Exception e : exceptions) { + System.err.println(Throwables.getStackTraceAsString(e)); + } + } + assertThat(exceptions).isEmpty(); + } } From 3b2db7cfa11d9daf6e326202f3909d9d104f95dd Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 12:37:36 +0200 Subject: [PATCH 211/349] project.clj: add test dependencies of the Java code in storm-starter --- project.clj | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 5afe4cc433..21c31ec869 100644 --- a/project.clj +++ b/project.clj @@ -15,6 +15,10 @@ :profiles {:dev {:dependencies [[storm "0.8.2"] - [org.clojure/clojure "1.4.0"]]}} + [org.clojure/clojure "1.4.0"] + [org.testng/testng "6.8.5"] + [org.easytesting/fest-assert-core "2.0M8"] + [org.mockito/mockito-all "1.9.0"] + [org.jmock/jmock "2.6.0"]]}} :min-lein-version "2.0.0" ) From a51c8247e3f358788efc8fb3ed7104b7b7cb21c4 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 12:39:14 +0200 Subject: [PATCH 212/349] Reformat Java code to use 2 instead of 4 spaces (to match Clojure style) --- src/jvm/storm/starter/BasicDRPCTopology.java | 84 ++-- .../storm/starter/ExclamationTopology.java | 82 ++-- src/jvm/storm/starter/ManualDRPC.java | 64 ++- src/jvm/storm/starter/ReachTopology.java | 297 ++++++------ src/jvm/storm/starter/RollingTopWords.java | 92 ++-- src/jvm/storm/starter/SingleJoinExample.java | 66 +-- .../starter/TransactionalGlobalCount.java | 237 ++++----- src/jvm/storm/starter/TransactionalWords.java | 369 +++++++------- src/jvm/storm/starter/WordCountTopology.java | 132 ++--- .../starter/bolt/AbstractRankerBolt.java | 126 +++-- .../bolt/IntermediateRankingsBolt.java | 46 +- src/jvm/storm/starter/bolt/PrinterBolt.java | 16 +- .../storm/starter/bolt/RollingCountBolt.java | 183 ++++--- .../storm/starter/bolt/SingleJoinBolt.java | 153 +++--- .../storm/starter/bolt/TotalRankingsBolt.java | 46 +- .../starter/spout/RandomSentenceSpout.java | 69 ++- .../tools/NthLastModifiedTimeTracker.java | 64 ++- src/jvm/storm/starter/tools/Rankable.java | 4 +- .../tools/RankableObjectWithFields.java | 189 ++++---- src/jvm/storm/starter/tools/Rankings.java | 142 +++--- .../starter/tools/SlidingWindowCounter.java | 111 +++-- .../storm/starter/tools/SlotBasedCounter.java | 138 +++--- .../storm/starter/trident/TridentReach.java | 199 ++++---- .../starter/trident/TridentWordCount.java | 98 ++-- src/jvm/storm/starter/util/StormRunner.java | 22 +- src/jvm/storm/starter/util/TupleHelpers.java | 12 +- .../bolt/IntermediateRankingsBoltTest.java | 238 +++++---- .../starter/bolt/RollingCountBoltTest.java | 171 ++++--- .../starter/bolt/TotalRankingsBoltTest.java | 241 +++++----- .../storm/starter/tools/MockTupleHelpers.java | 26 +- .../tools/NthLastModifiedTimeTrackerTest.java | 194 ++++---- .../tools/RankableObjectWithFieldsTest.java | 419 ++++++++-------- .../jvm/storm/starter/tools/RankingsTest.java | 453 +++++++++--------- .../tools/SlidingWindowCounterTest.java | 160 +++---- .../starter/tools/SlotBasedCounterTest.java | 304 ++++++------ 35 files changed, 2574 insertions(+), 2673 deletions(-) diff --git a/src/jvm/storm/starter/BasicDRPCTopology.java b/src/jvm/storm/starter/BasicDRPCTopology.java index 1ea8e7760b..7e7ef93a3b 100644 --- a/src/jvm/storm/starter/BasicDRPCTopology.java +++ b/src/jvm/storm/starter/BasicDRPCTopology.java @@ -5,59 +5,57 @@ import backtype.storm.LocalDRPC; import backtype.storm.StormSubmitter; import backtype.storm.drpc.LinearDRPCTopologyBuilder; -import backtype.storm.task.TopologyContext; import backtype.storm.topology.BasicOutputCollector; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseBasicBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; -import java.util.Map; /** - * This topology is a basic example of doing distributed RPC on top of Storm. It implements a function - * that appends a "!" to any string you send the DRPC function. - * - * See https://github.com/nathanmarz/storm/wiki/Distributed-RPC for more information on - * doing distributed RPC on top of Storm. + * This topology is a basic example of doing distributed RPC on top of Storm. It implements a function that appends a + * "!" to any string you send the DRPC function. + *

+ * See https://github.com/nathanmarz/storm/wiki/Distributed-RPC for more information on doing distributed RPC on top of + * Storm. */ public class BasicDRPCTopology { - public static class ExclaimBolt extends BaseBasicBolt { - @Override - public void execute(Tuple tuple, BasicOutputCollector collector) { - String input = tuple.getString(1); - collector.emit(new Values(tuple.getValue(0), input + "!")); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("id", "result")); - } - + public static class ExclaimBolt extends BaseBasicBolt { + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + String input = tuple.getString(1); + collector.emit(new Values(tuple.getValue(0), input + "!")); } - - public static void main(String[] args) throws Exception { - LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("exclamation"); - builder.addBolt(new ExclaimBolt(), 3); - - Config conf = new Config(); - - if(args==null || args.length==0) { - LocalDRPC drpc = new LocalDRPC(); - LocalCluster cluster = new LocalCluster(); - - cluster.submitTopology("drpc-demo", conf, builder.createLocalTopology(drpc)); - - for(String word: new String[] {"hello", "goodbye"}) { - System.out.println("Result for \"" + word + "\": " - + drpc.execute("exclamation", word)); - } - - cluster.shutdown(); - drpc.shutdown(); - } else { - conf.setNumWorkers(3); - StormSubmitter.submitTopology(args[0], conf, builder.createRemoteTopology()); - } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "result")); + } + + } + + public static void main(String[] args) throws Exception { + LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("exclamation"); + builder.addBolt(new ExclaimBolt(), 3); + + Config conf = new Config(); + + if (args == null || args.length == 0) { + LocalDRPC drpc = new LocalDRPC(); + LocalCluster cluster = new LocalCluster(); + + cluster.submitTopology("drpc-demo", conf, builder.createLocalTopology(drpc)); + + for (String word : new String[]{ "hello", "goodbye" }) { + System.out.println("Result for \"" + word + "\": " + drpc.execute("exclamation", word)); + } + + cluster.shutdown(); + drpc.shutdown(); + } + else { + conf.setNumWorkers(3); + StormSubmitter.submitTopology(args[0], conf, builder.createRemoteTopology()); } + } } diff --git a/src/jvm/storm/starter/ExclamationTopology.java b/src/jvm/storm/starter/ExclamationTopology.java index b98c8cd284..fed8b1e780 100644 --- a/src/jvm/storm/starter/ExclamationTopology.java +++ b/src/jvm/storm/starter/ExclamationTopology.java @@ -13,58 +13,58 @@ import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; import backtype.storm.utils.Utils; + import java.util.Map; /** * This is a basic example of a Storm topology. */ public class ExclamationTopology { - - public static class ExclamationBolt extends BaseRichBolt { - OutputCollector _collector; - @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { - _collector = collector; - } + public static class ExclamationBolt extends BaseRichBolt { + OutputCollector _collector; + + @Override + public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + _collector = collector; + } + + @Override + public void execute(Tuple tuple) { + _collector.emit(tuple, new Values(tuple.getString(0) + "!!!")); + _collector.ack(tuple); + } - @Override - public void execute(Tuple tuple) { - _collector.emit(tuple, new Values(tuple.getString(0) + "!!!")); - _collector.ack(tuple); - } + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("word")); + } - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("word")); - } + } + public static void main(String[] args) throws Exception { + TopologyBuilder builder = new TopologyBuilder(); + + builder.setSpout("word", new TestWordSpout(), 10); + builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word"); + builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1"); + + Config conf = new Config(); + conf.setDebug(true); + + if (args != null && args.length > 0) { + conf.setNumWorkers(3); + + StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); } - - public static void main(String[] args) throws Exception { - TopologyBuilder builder = new TopologyBuilder(); - - builder.setSpout("word", new TestWordSpout(), 10); - builder.setBolt("exclaim1", new ExclamationBolt(), 3) - .shuffleGrouping("word"); - builder.setBolt("exclaim2", new ExclamationBolt(), 2) - .shuffleGrouping("exclaim1"); - - Config conf = new Config(); - conf.setDebug(true); - - if(args!=null && args.length > 0) { - conf.setNumWorkers(3); - - StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); - } else { - - LocalCluster cluster = new LocalCluster(); - cluster.submitTopology("test", conf, builder.createTopology()); - Utils.sleep(10000); - cluster.killTopology("test"); - cluster.shutdown(); - } + else { + + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology("test", conf, builder.createTopology()); + Utils.sleep(10000); + cluster.killTopology("test"); + cluster.shutdown(); } + } } diff --git a/src/jvm/storm/starter/ManualDRPC.java b/src/jvm/storm/starter/ManualDRPC.java index fd45c9c0c6..ade4ab1908 100644 --- a/src/jvm/storm/starter/ManualDRPC.java +++ b/src/jvm/storm/starter/ManualDRPC.java @@ -15,39 +15,37 @@ public class ManualDRPC { - public static class ExclamationBolt extends BaseBasicBolt { - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("result", "return-info")); - } - - @Override - public void execute(Tuple tuple, BasicOutputCollector collector) { - String arg = tuple.getString(0); - Object retInfo = tuple.getValue(1); - collector.emit(new Values(arg + "!!!", retInfo)); - } - + public static class ExclamationBolt extends BaseBasicBolt { + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("result", "return-info")); } - - public static void main(String[] args) { - TopologyBuilder builder = new TopologyBuilder(); - LocalDRPC drpc = new LocalDRPC(); - - DRPCSpout spout = new DRPCSpout("exclamation", drpc); - builder.setSpout("drpc", spout); - builder.setBolt("exclaim", new ExclamationBolt(), 3) - .shuffleGrouping("drpc"); - builder.setBolt("return", new ReturnResults(), 3) - .shuffleGrouping("exclaim"); - - LocalCluster cluster = new LocalCluster(); - Config conf = new Config(); - cluster.submitTopology("exclaim", conf, builder.createTopology()); - - System.out.println(drpc.execute("exclamation", "aaa")); - System.out.println(drpc.execute("exclamation", "bbb")); - + + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + String arg = tuple.getString(0); + Object retInfo = tuple.getValue(1); + collector.emit(new Values(arg + "!!!", retInfo)); } + + } + + public static void main(String[] args) { + TopologyBuilder builder = new TopologyBuilder(); + LocalDRPC drpc = new LocalDRPC(); + + DRPCSpout spout = new DRPCSpout("exclamation", drpc); + builder.setSpout("drpc", spout); + builder.setBolt("exclaim", new ExclamationBolt(), 3).shuffleGrouping("drpc"); + builder.setBolt("return", new ReturnResults(), 3).shuffleGrouping("exclaim"); + + LocalCluster cluster = new LocalCluster(); + Config conf = new Config(); + cluster.submitTopology("exclaim", conf, builder.createTopology()); + + System.out.println(drpc.execute("exclamation", "aaa")); + System.out.println(drpc.execute("exclamation", "bbb")); + + } } diff --git a/src/jvm/storm/starter/ReachTopology.java b/src/jvm/storm/starter/ReachTopology.java index 8a7fdbcefc..a63db3e3fa 100644 --- a/src/jvm/storm/starter/ReachTopology.java +++ b/src/jvm/storm/starter/ReachTopology.java @@ -5,186 +5,175 @@ import backtype.storm.LocalDRPC; import backtype.storm.StormSubmitter; import backtype.storm.coordination.BatchOutputCollector; -import backtype.storm.coordination.CoordinatedBolt.FinishedCallback; import backtype.storm.drpc.LinearDRPCTopologyBuilder; -import backtype.storm.task.OutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.topology.BasicOutputCollector; -import backtype.storm.topology.IRichBolt; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseBasicBolt; import backtype.storm.topology.base.BaseBatchBolt; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; + +import java.util.*; /** - * This is a good example of doing complex Distributed RPC on top of Storm. This - * program creates a topology that can compute the reach for any URL on Twitter - * in realtime by parallelizing the whole computation. - * - * Reach is the number of unique people exposed to a URL on Twitter. To compute reach, - * you have to get all the people who tweeted the URL, get all the followers of all those people, - * unique that set of followers, and then count the unique set. It's an intense computation - * that can involve thousands of database calls and tens of millions of follower records. - * - * This Storm topology does every piece of that computation in parallel, turning what would be a - * computation that takes minutes on a single machine into one that takes just a couple seconds. - * - * For the purposes of demonstration, this topology replaces the use of actual DBs with - * in-memory hashmaps. - * + * This is a good example of doing complex Distributed RPC on top of Storm. This program creates a topology that can + * compute the reach for any URL on Twitter in realtime by parallelizing the whole computation. + *

+ * Reach is the number of unique people exposed to a URL on Twitter. To compute reach, you have to get all the people + * who tweeted the URL, get all the followers of all those people, unique that set of followers, and then count the + * unique set. It's an intense computation that can involve thousands of database calls and tens of millions of follower + * records. + *

+ * This Storm topology does every piece of that computation in parallel, turning what would be a computation that takes + * minutes on a single machine into one that takes just a couple seconds. + *

+ * For the purposes of demonstration, this topology replaces the use of actual DBs with in-memory hashmaps. + *

* See https://github.com/nathanmarz/storm/wiki/Distributed-RPC for more information on Distributed RPC. */ public class ReachTopology { - public static Map> TWEETERS_DB = new HashMap>() {{ - put("foo.com/blog/1", Arrays.asList("sally", "bob", "tim", "george", "nathan")); - put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally", "nathan")); - put("tech.backtype.com/blog/123", Arrays.asList("tim", "mike", "john")); - }}; - - public static Map> FOLLOWERS_DB = new HashMap>() {{ - put("sally", Arrays.asList("bob", "tim", "alice", "adam", "jim", "chris", "jai")); - put("bob", Arrays.asList("sally", "nathan", "jim", "mary", "david", "vivian")); - put("tim", Arrays.asList("alex")); - put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily", "jordan")); - put("adam", Arrays.asList("david", "carissa")); - put("mike", Arrays.asList("john", "bob")); - put("john", Arrays.asList("alice", "nathan", "jim", "mike", "bob")); - }}; - - public static class GetTweeters extends BaseBasicBolt { - @Override - public void execute(Tuple tuple, BasicOutputCollector collector) { - Object id = tuple.getValue(0); - String url = tuple.getString(1); - List tweeters = TWEETERS_DB.get(url); - if(tweeters!=null) { - for(String tweeter: tweeters) { - collector.emit(new Values(id, tweeter)); - } - } + public static Map> TWEETERS_DB = new HashMap>() {{ + put("foo.com/blog/1", Arrays.asList("sally", "bob", "tim", "george", "nathan")); + put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally", "nathan")); + put("tech.backtype.com/blog/123", Arrays.asList("tim", "mike", "john")); + }}; + + public static Map> FOLLOWERS_DB = new HashMap>() {{ + put("sally", Arrays.asList("bob", "tim", "alice", "adam", "jim", "chris", "jai")); + put("bob", Arrays.asList("sally", "nathan", "jim", "mary", "david", "vivian")); + put("tim", Arrays.asList("alex")); + put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily", "jordan")); + put("adam", Arrays.asList("david", "carissa")); + put("mike", Arrays.asList("john", "bob")); + put("john", Arrays.asList("alice", "nathan", "jim", "mike", "bob")); + }}; + + public static class GetTweeters extends BaseBasicBolt { + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + Object id = tuple.getValue(0); + String url = tuple.getString(1); + List tweeters = TWEETERS_DB.get(url); + if (tweeters != null) { + for (String tweeter : tweeters) { + collector.emit(new Values(id, tweeter)); } + } + } - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("id", "tweeter")); - } + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "tweeter")); } - - public static class GetFollowers extends BaseBasicBolt { - @Override - public void execute(Tuple tuple, BasicOutputCollector collector) { - Object id = tuple.getValue(0); - String tweeter = tuple.getString(1); - List followers = FOLLOWERS_DB.get(tweeter); - if(followers!=null) { - for(String follower: followers) { - collector.emit(new Values(id, follower)); - } - } - } + } - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("id", "follower")); + public static class GetFollowers extends BaseBasicBolt { + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + Object id = tuple.getValue(0); + String tweeter = tuple.getString(1); + List followers = FOLLOWERS_DB.get(tweeter); + if (followers != null) { + for (String follower : followers) { + collector.emit(new Values(id, follower)); } + } } - - public static class PartialUniquer extends BaseBatchBolt { - BatchOutputCollector _collector; - Object _id; - Set _followers = new HashSet(); - - @Override - public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { - _collector = collector; - _id = id; - } - @Override - public void execute(Tuple tuple) { - _followers.add(tuple.getString(1)); - } - - @Override - public void finishBatch() { - _collector.emit(new Values(_id, _followers.size())); - } + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "follower")); + } + } - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("id", "partial-count")); - } + public static class PartialUniquer extends BaseBatchBolt { + BatchOutputCollector _collector; + Object _id; + Set _followers = new HashSet(); + + @Override + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { + _collector = collector; + _id = id; } - - public static class CountAggregator extends BaseBatchBolt { - BatchOutputCollector _collector; - Object _id; - int _count = 0; - - @Override - public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { - _collector = collector; - _id = id; - } - @Override - public void execute(Tuple tuple) { - _count+=tuple.getInteger(1); - } - - @Override - public void finishBatch() { - _collector.emit(new Values(_id, _count)); - } + @Override + public void execute(Tuple tuple) { + _followers.add(tuple.getString(1)); + } - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("id", "reach")); - } + @Override + public void finishBatch() { + _collector.emit(new Values(_id, _followers.size())); } - - public static LinearDRPCTopologyBuilder construct() { - LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("reach"); - builder.addBolt(new GetTweeters(), 4); - builder.addBolt(new GetFollowers(), 12) - .shuffleGrouping(); - builder.addBolt(new PartialUniquer(), 6) - .fieldsGrouping(new Fields("id", "follower")); - builder.addBolt(new CountAggregator(), 3) - .fieldsGrouping(new Fields("id")); - return builder; + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "partial-count")); } - - public static void main(String[] args) throws Exception { - LinearDRPCTopologyBuilder builder = construct(); - - - Config conf = new Config(); - - if(args==null || args.length==0) { - conf.setMaxTaskParallelism(3); - LocalDRPC drpc = new LocalDRPC(); - LocalCluster cluster = new LocalCluster(); - cluster.submitTopology("reach-drpc", conf, builder.createLocalTopology(drpc)); - - String[] urlsToTry = new String[] { "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com"}; - for(String url: urlsToTry) { - System.out.println("Reach of " + url + ": " + drpc.execute("reach", url)); - } - - cluster.shutdown(); - drpc.shutdown(); - } else { - conf.setNumWorkers(6); - StormSubmitter.submitTopology(args[0], conf, builder.createRemoteTopology()); - } + } + + public static class CountAggregator extends BaseBatchBolt { + BatchOutputCollector _collector; + Object _id; + int _count = 0; + + @Override + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { + _collector = collector; + _id = id; + } + + @Override + public void execute(Tuple tuple) { + _count += tuple.getInteger(1); + } + + @Override + public void finishBatch() { + _collector.emit(new Values(_id, _count)); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "reach")); + } + } + + public static LinearDRPCTopologyBuilder construct() { + LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("reach"); + builder.addBolt(new GetTweeters(), 4); + builder.addBolt(new GetFollowers(), 12).shuffleGrouping(); + builder.addBolt(new PartialUniquer(), 6).fieldsGrouping(new Fields("id", "follower")); + builder.addBolt(new CountAggregator(), 3).fieldsGrouping(new Fields("id")); + return builder; + } + + public static void main(String[] args) throws Exception { + LinearDRPCTopologyBuilder builder = construct(); + + + Config conf = new Config(); + + if (args == null || args.length == 0) { + conf.setMaxTaskParallelism(3); + LocalDRPC drpc = new LocalDRPC(); + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology("reach-drpc", conf, builder.createLocalTopology(drpc)); + + String[] urlsToTry = new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" }; + for (String url : urlsToTry) { + System.out.println("Reach of " + url + ": " + drpc.execute("reach", url)); + } + + cluster.shutdown(); + drpc.shutdown(); + } + else { + conf.setNumWorkers(6); + StormSubmitter.submitTopology(args[0], conf, builder.createRemoteTopology()); } + } } diff --git a/src/jvm/storm/starter/RollingTopWords.java b/src/jvm/storm/starter/RollingTopWords.java index ef016a4daf..9d0b3c822d 100644 --- a/src/jvm/storm/starter/RollingTopWords.java +++ b/src/jvm/storm/starter/RollingTopWords.java @@ -1,13 +1,13 @@ package storm.starter; -import storm.starter.bolt.IntermediateRankingsBolt; -import storm.starter.bolt.RollingCountBolt; -import storm.starter.bolt.TotalRankingsBolt; -import storm.starter.util.StormRunner; import backtype.storm.Config; import backtype.storm.testing.TestWordSpout; import backtype.storm.topology.TopologyBuilder; import backtype.storm.tuple.Fields; +import storm.starter.bolt.IntermediateRankingsBolt; +import storm.starter.bolt.RollingCountBolt; +import storm.starter.bolt.TotalRankingsBolt; +import storm.starter.util.StormRunner; /** * This topology does a continuous computation of the top N words that the topology has seen in terms of cardinality. @@ -16,46 +16,46 @@ */ public class RollingTopWords { - private static final int DEFAULT_RUNTIME_IN_SECONDS = 60; - private static final int TOP_N = 5; - - private final TopologyBuilder builder; - private final String topologyName; - private final Config topologyConfig; - private final int runtimeInSeconds; - - public RollingTopWords() throws InterruptedException { - builder = new TopologyBuilder(); - topologyName = "slidingWindowCounts"; - topologyConfig = createTopologyConfiguration(); - runtimeInSeconds = DEFAULT_RUNTIME_IN_SECONDS; - - wireTopology(); - } - - private static Config createTopologyConfiguration() { - Config conf = new Config(); - conf.setDebug(true); - return conf; - } - - private void wireTopology() throws InterruptedException { - String spoutId = "wordGenerator"; - String counterId = "counter"; - String intermediateRankerId = "intermediateRanker"; - String totalRankerId = "finalRanker"; - builder.setSpout(spoutId, new TestWordSpout(), 5); - builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word")); - builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId, - new Fields("obj")); - builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId); - } - - public void run() throws InterruptedException { - StormRunner.runTopologyLocally(builder.createTopology(), topologyName, topologyConfig, runtimeInSeconds); - } - - public static void main(String[] args) throws Exception { - new RollingTopWords().run(); - } + private static final int DEFAULT_RUNTIME_IN_SECONDS = 60; + private static final int TOP_N = 5; + + private final TopologyBuilder builder; + private final String topologyName; + private final Config topologyConfig; + private final int runtimeInSeconds; + + public RollingTopWords() throws InterruptedException { + builder = new TopologyBuilder(); + topologyName = "slidingWindowCounts"; + topologyConfig = createTopologyConfiguration(); + runtimeInSeconds = DEFAULT_RUNTIME_IN_SECONDS; + + wireTopology(); + } + + private static Config createTopologyConfiguration() { + Config conf = new Config(); + conf.setDebug(true); + return conf; + } + + private void wireTopology() throws InterruptedException { + String spoutId = "wordGenerator"; + String counterId = "counter"; + String intermediateRankerId = "intermediateRanker"; + String totalRankerId = "finalRanker"; + builder.setSpout(spoutId, new TestWordSpout(), 5); + builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word")); + builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId, new Fields( + "obj")); + builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId); + } + + public void run() throws InterruptedException { + StormRunner.runTopologyLocally(builder.createTopology(), topologyName, topologyConfig, runtimeInSeconds); + } + + public static void main(String[] args) throws Exception { + new RollingTopWords().run(); + } } diff --git a/src/jvm/storm/starter/SingleJoinExample.java b/src/jvm/storm/starter/SingleJoinExample.java index 62d332ea80..d323809c1b 100644 --- a/src/jvm/storm/starter/SingleJoinExample.java +++ b/src/jvm/storm/starter/SingleJoinExample.java @@ -10,38 +10,38 @@ import storm.starter.bolt.SingleJoinBolt; public class SingleJoinExample { - public static void main(String[] args) { - FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender")); - FeederSpout ageSpout = new FeederSpout(new Fields("id", "age")); - - TopologyBuilder builder = new TopologyBuilder(); - builder.setSpout("gender", genderSpout); - builder.setSpout("age", ageSpout); - builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age"))) - .fieldsGrouping("gender", new Fields("id")) - .fieldsGrouping("age", new Fields("id")); - - Config conf = new Config(); - conf.setDebug(true); - - LocalCluster cluster = new LocalCluster(); - cluster.submitTopology("join-example", conf, builder.createTopology()); - - for(int i=0; i<10; i++) { - String gender; - if(i % 2 == 0) { - gender = "male"; - } else { - gender = "female"; - } - genderSpout.feed(new Values(i, gender)); - } - - for(int i=9; i>=0; i--) { - ageSpout.feed(new Values(i, i+20)); - } - - Utils.sleep(2000); - cluster.shutdown(); + public static void main(String[] args) { + FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender")); + FeederSpout ageSpout = new FeederSpout(new Fields("id", "age")); + + TopologyBuilder builder = new TopologyBuilder(); + builder.setSpout("gender", genderSpout); + builder.setSpout("age", ageSpout); + builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age"))).fieldsGrouping("gender", new Fields("id")) + .fieldsGrouping("age", new Fields("id")); + + Config conf = new Config(); + conf.setDebug(true); + + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology("join-example", conf, builder.createTopology()); + + for (int i = 0; i < 10; i++) { + String gender; + if (i % 2 == 0) { + gender = "male"; + } + else { + gender = "female"; + } + genderSpout.feed(new Values(i, gender)); } + + for (int i = 9; i >= 0; i--) { + ageSpout.feed(new Values(i, i + 20)); + } + + Utils.sleep(2000); + cluster.shutdown(); + } } diff --git a/src/jvm/storm/starter/TransactionalGlobalCount.java b/src/jvm/storm/starter/TransactionalGlobalCount.java index b4579a4e13..91b16aab7b 100644 --- a/src/jvm/storm/starter/TransactionalGlobalCount.java +++ b/src/jvm/storm/starter/TransactionalGlobalCount.java @@ -14,6 +14,7 @@ import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; + import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; @@ -21,135 +22,135 @@ import java.util.Map; /** - * This is a basic example of a transactional topology. It keeps a count of the number of tuples seen so far - * in a database. The source of data and the databases are mocked out as in memory maps for demonstration - * purposes. This class is defined in depth on the wiki at https://github.com/nathanmarz/storm/wiki/Transactional-topologies + * This is a basic example of a transactional topology. It keeps a count of the number of tuples seen so far in a + * database. The source of data and the databases are mocked out as in memory maps for demonstration purposes. This + * class is defined in depth on the wiki at https://github.com/nathanmarz/storm/wiki/Transactional-topologies */ public class TransactionalGlobalCount { - public static final int PARTITION_TAKE_PER_BATCH = 3; - public static final Map>> DATA = new HashMap>>() {{ - put(0, new ArrayList>() {{ - add(new Values("cat")); - add(new Values("dog")); - add(new Values("chicken")); - add(new Values("cat")); - add(new Values("dog")); - add(new Values("apple")); - }}); - put(1, new ArrayList>() {{ - add(new Values("cat")); - add(new Values("dog")); - add(new Values("apple")); - add(new Values("banana")); - }}); - put(2, new ArrayList>() {{ - add(new Values("cat")); - add(new Values("cat")); - add(new Values("cat")); - add(new Values("cat")); - add(new Values("cat")); - add(new Values("dog")); - add(new Values("dog")); - add(new Values("dog")); - add(new Values("dog")); - }}); - }}; - - public static class Value { - int count = 0; - BigInteger txid; + public static final int PARTITION_TAKE_PER_BATCH = 3; + public static final Map>> DATA = new HashMap>>() {{ + put(0, new ArrayList>() {{ + add(new Values("cat")); + add(new Values("dog")); + add(new Values("chicken")); + add(new Values("cat")); + add(new Values("dog")); + add(new Values("apple")); + }}); + put(1, new ArrayList>() {{ + add(new Values("cat")); + add(new Values("dog")); + add(new Values("apple")); + add(new Values("banana")); + }}); + put(2, new ArrayList>() {{ + add(new Values("cat")); + add(new Values("cat")); + add(new Values("cat")); + add(new Values("cat")); + add(new Values("cat")); + add(new Values("dog")); + add(new Values("dog")); + add(new Values("dog")); + add(new Values("dog")); + }}); + }}; + + public static class Value { + int count = 0; + BigInteger txid; + } + + public static Map DATABASE = new HashMap(); + public static final String GLOBAL_COUNT_KEY = "GLOBAL-COUNT"; + + public static class BatchCount extends BaseBatchBolt { + Object _id; + BatchOutputCollector _collector; + + int _count = 0; + + @Override + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { + _collector = collector; + _id = id; } - public static Map DATABASE = new HashMap(); - public static final String GLOBAL_COUNT_KEY = "GLOBAL-COUNT"; - - public static class BatchCount extends BaseBatchBolt { - Object _id; - BatchOutputCollector _collector; - - int _count = 0; - - @Override - public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { - _collector = collector; - _id = id; - } + @Override + public void execute(Tuple tuple) { + _count++; + } - @Override - public void execute(Tuple tuple) { - _count++; - } + @Override + public void finishBatch() { + _collector.emit(new Values(_id, _count)); + } - @Override - public void finishBatch() { - _collector.emit(new Values(_id, _count)); - } + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "count")); + } + } + + public static class UpdateGlobalCount extends BaseTransactionalBolt implements ICommitter { + TransactionAttempt _attempt; + BatchOutputCollector _collector; - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("id", "count")); - } + int _sum = 0; + + @Override + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, TransactionAttempt attempt) { + _collector = collector; + _attempt = attempt; } - - public static class UpdateGlobalCount extends BaseTransactionalBolt implements ICommitter { - TransactionAttempt _attempt; - BatchOutputCollector _collector; - - int _sum = 0; - - @Override - public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, TransactionAttempt attempt) { - _collector = collector; - _attempt = attempt; - } - @Override - public void execute(Tuple tuple) { - _sum+=tuple.getInteger(1); - } + @Override + public void execute(Tuple tuple) { + _sum += tuple.getInteger(1); + } - @Override - public void finishBatch() { - Value val = DATABASE.get(GLOBAL_COUNT_KEY); - Value newval; - if(val == null || !val.txid.equals(_attempt.getTransactionId())) { - newval = new Value(); - newval.txid = _attempt.getTransactionId(); - if(val==null) { - newval.count = _sum; - } else { - newval.count = _sum + val.count; - } - DATABASE.put(GLOBAL_COUNT_KEY, newval); - } else { - newval = val; - } - _collector.emit(new Values(_attempt, newval.count)); + @Override + public void finishBatch() { + Value val = DATABASE.get(GLOBAL_COUNT_KEY); + Value newval; + if (val == null || !val.txid.equals(_attempt.getTransactionId())) { + newval = new Value(); + newval.txid = _attempt.getTransactionId(); + if (val == null) { + newval.count = _sum; } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("id", "sum")); - } + else { + newval.count = _sum + val.count; + } + DATABASE.put(GLOBAL_COUNT_KEY, newval); + } + else { + newval = val; + } + _collector.emit(new Values(_attempt, newval.count)); } - - public static void main(String[] args) throws Exception { - MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH); - TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 3); - builder.setBolt("partial-count", new BatchCount(), 5) - .noneGrouping("spout"); - builder.setBolt("sum", new UpdateGlobalCount()) - .globalGrouping("partial-count"); - - LocalCluster cluster = new LocalCluster(); - - Config config = new Config(); - config.setDebug(true); - config.setMaxSpoutPending(3); - - cluster.submitTopology("global-count-topology", config, builder.buildTopology()); - - Thread.sleep(3000); - cluster.shutdown(); + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "sum")); } + } + + public static void main(String[] args) throws Exception { + MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH); + TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 3); + builder.setBolt("partial-count", new BatchCount(), 5).noneGrouping("spout"); + builder.setBolt("sum", new UpdateGlobalCount()).globalGrouping("partial-count"); + + LocalCluster cluster = new LocalCluster(); + + Config config = new Config(); + config.setDebug(true); + config.setMaxSpoutPending(3); + + cluster.submitTopology("global-count-topology", config, builder.buildTopology()); + + Thread.sleep(3000); + cluster.shutdown(); + } } diff --git a/src/jvm/storm/starter/TransactionalWords.java b/src/jvm/storm/starter/TransactionalWords.java index 0252b66134..4ee7b1271d 100644 --- a/src/jvm/storm/starter/TransactionalWords.java +++ b/src/jvm/storm/starter/TransactionalWords.java @@ -15,6 +15,7 @@ import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; + import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; @@ -22,205 +23,207 @@ import java.util.Map; /** - * This class defines a more involved transactional topology then TransactionalGlobalCount. This topology - * processes a stream of words and produces two outputs: - * - * 1. A count for each word (stored in a database) - * 2. The number of words for every bucket of 10 counts. So it stores in the database how many words have appeared - * 0-9 times, how many have appeared 10-19 times, and so on. - * + * This class defines a more involved transactional topology then TransactionalGlobalCount. This topology processes a + * stream of words and produces two outputs: + *

+ * 1. A count for each word (stored in a database) 2. The number of words for every bucket of 10 counts. So it stores in + * the database how many words have appeared 0-9 times, how many have appeared 10-19 times, and so on. + *

* A batch of words can cause the bucket counts to decrement for some buckets and increment for others as words move * between buckets as their counts accumulate. */ public class TransactionalWords { - public static class CountValue { - Integer prev_count = null; - int count = 0; - BigInteger txid = null; + public static class CountValue { + Integer prev_count = null; + int count = 0; + BigInteger txid = null; + } + + public static class BucketValue { + int count = 0; + BigInteger txid; + } + + public static final int BUCKET_SIZE = 10; + + public static Map COUNT_DATABASE = new HashMap(); + public static Map BUCKET_DATABASE = new HashMap(); + + + public static final int PARTITION_TAKE_PER_BATCH = 3; + + public static final Map>> DATA = new HashMap>>() {{ + put(0, new ArrayList>() {{ + add(new Values("cat")); + add(new Values("dog")); + add(new Values("chicken")); + add(new Values("cat")); + add(new Values("dog")); + add(new Values("apple")); + }}); + put(1, new ArrayList>() {{ + add(new Values("cat")); + add(new Values("dog")); + add(new Values("apple")); + add(new Values("banana")); + }}); + put(2, new ArrayList>() {{ + add(new Values("cat")); + add(new Values("cat")); + add(new Values("cat")); + add(new Values("cat")); + add(new Values("cat")); + add(new Values("dog")); + add(new Values("dog")); + add(new Values("dog")); + add(new Values("dog")); + }}); + }}; + + public static class KeyedCountUpdater extends BaseTransactionalBolt implements ICommitter { + Map _counts = new HashMap(); + BatchOutputCollector _collector; + TransactionAttempt _id; + + int _count = 0; + + @Override + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, TransactionAttempt id) { + _collector = collector; + _id = id; } - public static class BucketValue { - int count = 0; - BigInteger txid; + @Override + public void execute(Tuple tuple) { + String key = tuple.getString(1); + Integer curr = _counts.get(key); + if (curr == null) + curr = 0; + _counts.put(key, curr + 1); } - - public static final int BUCKET_SIZE = 10; - - public static Map COUNT_DATABASE = new HashMap(); - public static Map BUCKET_DATABASE = new HashMap(); - - - public static final int PARTITION_TAKE_PER_BATCH = 3; - - public static final Map>> DATA = new HashMap>>() {{ - put(0, new ArrayList>() {{ - add(new Values("cat")); - add(new Values("dog")); - add(new Values("chicken")); - add(new Values("cat")); - add(new Values("dog")); - add(new Values("apple")); - }}); - put(1, new ArrayList>() {{ - add(new Values("cat")); - add(new Values("dog")); - add(new Values("apple")); - add(new Values("banana")); - }}); - put(2, new ArrayList>() {{ - add(new Values("cat")); - add(new Values("cat")); - add(new Values("cat")); - add(new Values("cat")); - add(new Values("cat")); - add(new Values("dog")); - add(new Values("dog")); - add(new Values("dog")); - add(new Values("dog")); - }}); - }}; - - public static class KeyedCountUpdater extends BaseTransactionalBolt implements ICommitter { - Map _counts = new HashMap(); - BatchOutputCollector _collector; - TransactionAttempt _id; - - int _count = 0; - - @Override - public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, TransactionAttempt id) { - _collector = collector; - _id = id; - } - @Override - public void execute(Tuple tuple) { - String key = tuple.getString(1); - Integer curr = _counts.get(key); - if(curr==null) curr = 0; - _counts.put(key, curr + 1); + @Override + public void finishBatch() { + for (String key : _counts.keySet()) { + CountValue val = COUNT_DATABASE.get(key); + CountValue newVal; + if (val == null || !val.txid.equals(_id)) { + newVal = new CountValue(); + newVal.txid = _id.getTransactionId(); + if (val != null) { + newVal.prev_count = val.count; + newVal.count = val.count; + } + newVal.count = newVal.count + _counts.get(key); + COUNT_DATABASE.put(key, newVal); } - - @Override - public void finishBatch() { - for(String key: _counts.keySet()) { - CountValue val = COUNT_DATABASE.get(key); - CountValue newVal; - if(val==null || !val.txid.equals(_id)) { - newVal = new CountValue(); - newVal.txid = _id.getTransactionId(); - if(val!=null) { - newVal.prev_count = val.count; - newVal.count = val.count; - } - newVal.count = newVal.count + _counts.get(key); - COUNT_DATABASE.put(key, newVal); - } else { - newVal = val; - } - _collector.emit(new Values(_id, key, newVal.count, newVal.prev_count)); - } + else { + newVal = val; } + _collector.emit(new Values(_id, key, newVal.count, newVal.prev_count)); + } + } - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("id", "key", "count", "prev-count")); - } + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "key", "count", "prev-count")); } - - public static class Bucketize extends BaseBasicBolt { - @Override - public void execute(Tuple tuple, BasicOutputCollector collector) { - TransactionAttempt attempt = (TransactionAttempt) tuple.getValue(0); - int curr = tuple.getInteger(2); - Integer prev = tuple.getInteger(3); - - int currBucket = curr / BUCKET_SIZE; - Integer prevBucket = null; - if(prev!=null) { - prevBucket = prev / BUCKET_SIZE; - } - - if(prevBucket==null) { - collector.emit(new Values(attempt, currBucket, 1)); - } else if(currBucket != prevBucket) { - collector.emit(new Values(attempt, currBucket, 1)); - collector.emit(new Values(attempt, prevBucket, -1)); - } - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("attempt", "bucket", "delta")); - } + } + + public static class Bucketize extends BaseBasicBolt { + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + TransactionAttempt attempt = (TransactionAttempt) tuple.getValue(0); + int curr = tuple.getInteger(2); + Integer prev = tuple.getInteger(3); + + int currBucket = curr / BUCKET_SIZE; + Integer prevBucket = null; + if (prev != null) { + prevBucket = prev / BUCKET_SIZE; + } + + if (prevBucket == null) { + collector.emit(new Values(attempt, currBucket, 1)); + } + else if (currBucket != prevBucket) { + collector.emit(new Values(attempt, currBucket, 1)); + collector.emit(new Values(attempt, prevBucket, -1)); + } } - - public static class BucketCountUpdater extends BaseTransactionalBolt { - Map _accum = new HashMap(); - BatchOutputCollector _collector; - TransactionAttempt _attempt; - - int _count = 0; - - @Override - public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, TransactionAttempt attempt) { - _collector = collector; - _attempt = attempt; - } - @Override - public void execute(Tuple tuple) { - Integer bucket = tuple.getInteger(1); - Integer delta = tuple.getInteger(2); - Integer curr = _accum.get(bucket); - if(curr==null) curr = 0; - _accum.put(bucket, curr + delta); - } + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("attempt", "bucket", "delta")); + } + } - @Override - public void finishBatch() { - for(Integer bucket: _accum.keySet()) { - BucketValue currVal = BUCKET_DATABASE.get(bucket); - BucketValue newVal; - if(currVal==null || !currVal.txid.equals(_attempt.getTransactionId())) { - newVal = new BucketValue(); - newVal.txid = _attempt.getTransactionId(); - newVal.count = _accum.get(bucket); - if(currVal!=null) newVal.count += currVal.count; - BUCKET_DATABASE.put(bucket, newVal); - } else { - newVal = currVal; - } - _collector.emit(new Values(_attempt, bucket, newVal.count)); - } - } + public static class BucketCountUpdater extends BaseTransactionalBolt { + Map _accum = new HashMap(); + BatchOutputCollector _collector; + TransactionAttempt _attempt; + + int _count = 0; + + @Override + public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, TransactionAttempt attempt) { + _collector = collector; + _attempt = attempt; + } + + @Override + public void execute(Tuple tuple) { + Integer bucket = tuple.getInteger(1); + Integer delta = tuple.getInteger(2); + Integer curr = _accum.get(bucket); + if (curr == null) + curr = 0; + _accum.put(bucket, curr + delta); + } - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("id", "bucket", "count")); - } + @Override + public void finishBatch() { + for (Integer bucket : _accum.keySet()) { + BucketValue currVal = BUCKET_DATABASE.get(bucket); + BucketValue newVal; + if (currVal == null || !currVal.txid.equals(_attempt.getTransactionId())) { + newVal = new BucketValue(); + newVal.txid = _attempt.getTransactionId(); + newVal.count = _accum.get(bucket); + if (currVal != null) + newVal.count += currVal.count; + BUCKET_DATABASE.put(bucket, newVal); + } + else { + newVal = currVal; + } + _collector.emit(new Values(_attempt, bucket, newVal.count)); + } } - - public static void main(String[] args) throws Exception { - MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH); - TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("top-n-words", "spout", spout, 2); - builder.setBolt("count", new KeyedCountUpdater(), 5) - .fieldsGrouping("spout", new Fields("word")); - builder.setBolt("bucketize", new Bucketize()) - .noneGrouping("count"); - builder.setBolt("buckets", new BucketCountUpdater(), 5) - .fieldsGrouping("bucketize", new Fields("bucket")); - - - LocalCluster cluster = new LocalCluster(); - - Config config = new Config(); - config.setDebug(true); - config.setMaxSpoutPending(3); - - cluster.submitTopology("top-n-topology", config, builder.buildTopology()); - - Thread.sleep(3000); - cluster.shutdown(); + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("id", "bucket", "count")); } + } + + public static void main(String[] args) throws Exception { + MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH); + TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("top-n-words", "spout", spout, 2); + builder.setBolt("count", new KeyedCountUpdater(), 5).fieldsGrouping("spout", new Fields("word")); + builder.setBolt("bucketize", new Bucketize()).noneGrouping("count"); + builder.setBolt("buckets", new BucketCountUpdater(), 5).fieldsGrouping("bucketize", new Fields("bucket")); + + + LocalCluster cluster = new LocalCluster(); + + Config config = new Config(); + config.setDebug(true); + config.setMaxSpoutPending(3); + + cluster.submitTopology("top-n-topology", config, builder.buildTopology()); + + Thread.sleep(3000); + cluster.shutdown(); + } } diff --git a/src/jvm/storm/starter/WordCountTopology.java b/src/jvm/storm/starter/WordCountTopology.java index bed968d908..f26278cfaa 100644 --- a/src/jvm/storm/starter/WordCountTopology.java +++ b/src/jvm/storm/starter/WordCountTopology.java @@ -1,11 +1,9 @@ package storm.starter; -import storm.starter.spout.RandomSentenceSpout; import backtype.storm.Config; import backtype.storm.LocalCluster; import backtype.storm.StormSubmitter; import backtype.storm.task.ShellBolt; -import backtype.storm.task.TopologyContext; import backtype.storm.topology.BasicOutputCollector; import backtype.storm.topology.IRichBolt; import backtype.storm.topology.OutputFieldsDeclarer; @@ -14,6 +12,8 @@ import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; +import storm.starter.spout.RandomSentenceSpout; + import java.util.HashMap; import java.util.Map; @@ -21,70 +21,70 @@ * This topology demonstrates Storm's stream groupings and multilang capabilities. */ public class WordCountTopology { - public static class SplitSentence extends ShellBolt implements IRichBolt { - - public SplitSentence() { - super("python", "splitsentence.py"); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("word")); - } - - @Override - public Map getComponentConfiguration() { - return null; - } - } - - public static class WordCount extends BaseBasicBolt { - Map counts = new HashMap(); - - @Override - public void execute(Tuple tuple, BasicOutputCollector collector) { - String word = tuple.getString(0); - Integer count = counts.get(word); - if(count==null) count = 0; - count++; - counts.put(word, count); - collector.emit(new Values(word, count)); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("word", "count")); - } + public static class SplitSentence extends ShellBolt implements IRichBolt { + + public SplitSentence() { + super("python", "splitsentence.py"); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("word")); + } + + @Override + public Map getComponentConfiguration() { + return null; + } + } + + public static class WordCount extends BaseBasicBolt { + Map counts = new HashMap(); + + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + String word = tuple.getString(0); + Integer count = counts.get(word); + if (count == null) + count = 0; + count++; + counts.put(word, count); + collector.emit(new Values(word, count)); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("word", "count")); } - - public static void main(String[] args) throws Exception { - - TopologyBuilder builder = new TopologyBuilder(); - - builder.setSpout("spout", new RandomSentenceSpout(), 5); - - builder.setBolt("split", new SplitSentence(), 8) - .shuffleGrouping("spout"); - builder.setBolt("count", new WordCount(), 12) - .fieldsGrouping("split", new Fields("word")); - - Config conf = new Config(); - conf.setDebug(true); - - - if(args!=null && args.length > 0) { - conf.setNumWorkers(3); - - StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); - } else { - conf.setMaxTaskParallelism(3); - - LocalCluster cluster = new LocalCluster(); - cluster.submitTopology("word-count", conf, builder.createTopology()); - - Thread.sleep(10000); - - cluster.shutdown(); - } + } + + public static void main(String[] args) throws Exception { + + TopologyBuilder builder = new TopologyBuilder(); + + builder.setSpout("spout", new RandomSentenceSpout(), 5); + + builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout"); + builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word")); + + Config conf = new Config(); + conf.setDebug(true); + + + if (args != null && args.length > 0) { + conf.setNumWorkers(3); + + StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); + } + else { + conf.setMaxTaskParallelism(3); + + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology("word-count", conf, builder.createTopology()); + + Thread.sleep(10000); + + cluster.shutdown(); } + } } diff --git a/src/jvm/storm/starter/bolt/AbstractRankerBolt.java b/src/jvm/storm/starter/bolt/AbstractRankerBolt.java index 27d65a0ce3..6e2551c966 100644 --- a/src/jvm/storm/starter/bolt/AbstractRankerBolt.java +++ b/src/jvm/storm/starter/bolt/AbstractRankerBolt.java @@ -1,12 +1,5 @@ package storm.starter.bolt; -import java.util.HashMap; -import java.util.Map; - -import org.apache.log4j.Logger; - -import storm.starter.tools.Rankings; -import storm.starter.util.TupleHelpers; import backtype.storm.Config; import backtype.storm.topology.BasicOutputCollector; import backtype.storm.topology.OutputFieldsDeclarer; @@ -14,82 +7,87 @@ import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; +import org.apache.log4j.Logger; +import storm.starter.tools.Rankings; +import storm.starter.util.TupleHelpers; + +import java.util.HashMap; +import java.util.Map; /** * This abstract bolt provides the basic behavior of bolts that rank objects according to their count. - * + *

* It uses a template method design pattern for {@link AbstractRankerBolt#execute(Tuple, BasicOutputCollector)} to allow * actual bolt implementations to specify how incoming tuples are processed, i.e. how the objects embedded within those * tuples are retrieved and counted. - * */ public abstract class AbstractRankerBolt extends BaseBasicBolt { - private static final long serialVersionUID = 4931640198501530202L; - private static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS = 2; - private static final int DEFAULT_COUNT = 10; + private static final long serialVersionUID = 4931640198501530202L; + private static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS = 2; + private static final int DEFAULT_COUNT = 10; - private final int emitFrequencyInSeconds; - private final int count; - private final Rankings rankings; + private final int emitFrequencyInSeconds; + private final int count; + private final Rankings rankings; - public AbstractRankerBolt() { - this(DEFAULT_COUNT, DEFAULT_EMIT_FREQUENCY_IN_SECONDS); - } + public AbstractRankerBolt() { + this(DEFAULT_COUNT, DEFAULT_EMIT_FREQUENCY_IN_SECONDS); + } - public AbstractRankerBolt(int topN) { - this(topN, DEFAULT_EMIT_FREQUENCY_IN_SECONDS); - } + public AbstractRankerBolt(int topN) { + this(topN, DEFAULT_EMIT_FREQUENCY_IN_SECONDS); + } - public AbstractRankerBolt(int topN, int emitFrequencyInSeconds) { - if (topN < 1) { - throw new IllegalArgumentException("topN must be >= 1 (you requested " + topN + ")"); - } - if (emitFrequencyInSeconds < 1) { - throw new IllegalArgumentException("The emit frequency must be >= 1 seconds (you requested " - + emitFrequencyInSeconds + " seconds)"); - } - count = topN; - this.emitFrequencyInSeconds = emitFrequencyInSeconds; - rankings = new Rankings(count); + public AbstractRankerBolt(int topN, int emitFrequencyInSeconds) { + if (topN < 1) { + throw new IllegalArgumentException("topN must be >= 1 (you requested " + topN + ")"); } - - protected Rankings getRankings() { - return rankings; + if (emitFrequencyInSeconds < 1) { + throw new IllegalArgumentException( + "The emit frequency must be >= 1 seconds (you requested " + emitFrequencyInSeconds + " seconds)"); } - - /** - * This method functions as a template method (design pattern). - */ - @Override - public final void execute(Tuple tuple, BasicOutputCollector collector) { - if (TupleHelpers.isTickTuple(tuple)) { - getLogger().debug("Received tick tuple, triggering emit of current rankings"); - emitRankings(collector); - } - else { - updateRankingsWithTuple(tuple); - } + count = topN; + this.emitFrequencyInSeconds = emitFrequencyInSeconds; + rankings = new Rankings(count); + } + + protected Rankings getRankings() { + return rankings; + } + + /** + * This method functions as a template method (design pattern). + */ + @Override + public final void execute(Tuple tuple, BasicOutputCollector collector) { + if (TupleHelpers.isTickTuple(tuple)) { + getLogger().debug("Received tick tuple, triggering emit of current rankings"); + emitRankings(collector); + } + else { + updateRankingsWithTuple(tuple); } + } - abstract void updateRankingsWithTuple(Tuple tuple); + abstract void updateRankingsWithTuple(Tuple tuple); - private void emitRankings(BasicOutputCollector collector) { - collector.emit(new Values(rankings)); - getLogger().debug("Rankings: " + rankings); - } + private void emitRankings(BasicOutputCollector collector) { + collector.emit(new Values(rankings)); + getLogger().debug("Rankings: " + rankings); + } - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("rankings")); - } + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("rankings")); + } - @Override - public Map getComponentConfiguration() { - Map conf = new HashMap(); - conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, emitFrequencyInSeconds); - return conf; - } + @Override + public Map getComponentConfiguration() { + Map conf = new HashMap(); + conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, emitFrequencyInSeconds); + return conf; + } - abstract Logger getLogger(); + abstract Logger getLogger(); } diff --git a/src/jvm/storm/starter/bolt/IntermediateRankingsBolt.java b/src/jvm/storm/starter/bolt/IntermediateRankingsBolt.java index 161e67d0ff..6a57d47028 100644 --- a/src/jvm/storm/starter/bolt/IntermediateRankingsBolt.java +++ b/src/jvm/storm/starter/bolt/IntermediateRankingsBolt.java @@ -1,43 +1,41 @@ package storm.starter.bolt; +import backtype.storm.tuple.Tuple; import org.apache.log4j.Logger; - import storm.starter.tools.Rankable; import storm.starter.tools.RankableObjectWithFields; -import backtype.storm.tuple.Tuple; /** * This bolt ranks incoming objects by their count. - * + *

* It assumes the input tuples to adhere to the following format: (object, object_count, additionalField1, * additionalField2, ..., additionalFieldN). - * */ public final class IntermediateRankingsBolt extends AbstractRankerBolt { - private static final long serialVersionUID = -1369800530256637409L; - private static final Logger LOG = Logger.getLogger(IntermediateRankingsBolt.class); + private static final long serialVersionUID = -1369800530256637409L; + private static final Logger LOG = Logger.getLogger(IntermediateRankingsBolt.class); - public IntermediateRankingsBolt() { - super(); - } + public IntermediateRankingsBolt() { + super(); + } - public IntermediateRankingsBolt(int topN) { - super(topN); - } + public IntermediateRankingsBolt(int topN) { + super(topN); + } - public IntermediateRankingsBolt(int topN, int emitFrequencyInSeconds) { - super(topN, emitFrequencyInSeconds); - } + public IntermediateRankingsBolt(int topN, int emitFrequencyInSeconds) { + super(topN, emitFrequencyInSeconds); + } - @Override - void updateRankingsWithTuple(Tuple tuple) { - Rankable rankable = RankableObjectWithFields.from(tuple); - super.getRankings().updateWith(rankable); - } + @Override + void updateRankingsWithTuple(Tuple tuple) { + Rankable rankable = RankableObjectWithFields.from(tuple); + super.getRankings().updateWith(rankable); + } - @Override - Logger getLogger() { - return LOG; - } + @Override + Logger getLogger() { + return LOG; + } } diff --git a/src/jvm/storm/starter/bolt/PrinterBolt.java b/src/jvm/storm/starter/bolt/PrinterBolt.java index ac380ed6a2..8b2e62bbf3 100644 --- a/src/jvm/storm/starter/bolt/PrinterBolt.java +++ b/src/jvm/storm/starter/bolt/PrinterBolt.java @@ -8,13 +8,13 @@ public class PrinterBolt extends BaseBasicBolt { - @Override - public void execute(Tuple tuple, BasicOutputCollector collector) { - System.out.println(tuple); - } + @Override + public void execute(Tuple tuple, BasicOutputCollector collector) { + System.out.println(tuple); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer ofd) { + } - @Override - public void declareOutputFields(OutputFieldsDeclarer ofd) { - } - } diff --git a/src/jvm/storm/starter/bolt/RollingCountBolt.java b/src/jvm/storm/starter/bolt/RollingCountBolt.java index 0066b61b8e..d5477496f3 100644 --- a/src/jvm/storm/starter/bolt/RollingCountBolt.java +++ b/src/jvm/storm/starter/bolt/RollingCountBolt.java @@ -1,14 +1,5 @@ package storm.starter.bolt; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; - -import org.apache.log4j.Logger; - -import storm.starter.tools.NthLastModifiedTimeTracker; -import storm.starter.tools.SlidingWindowCounter; -import storm.starter.util.TupleHelpers; import backtype.storm.Config; import backtype.storm.task.OutputCollector; import backtype.storm.task.TopologyContext; @@ -17,110 +8,118 @@ import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; +import org.apache.log4j.Logger; +import storm.starter.tools.NthLastModifiedTimeTracker; +import storm.starter.tools.SlidingWindowCounter; +import storm.starter.util.TupleHelpers; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; /** * This bolt performs rolling counts of incoming objects, i.e. sliding window based counting. - * + *

* The bolt is configured by two parameters, the length of the sliding window in seconds (which influences the output * data of the bolt, i.e. how it will count objects) and the emit frequency in seconds (which influences how often the * bolt will output the latest window counts). For instance, if the window length is set to an equivalent of five * minutes and the emit frequency to one minute, then the bolt will output the latest five-minute sliding window every * minute. - * + *

* The bolt emits a rolling count tuple per object, consisting of the object itself, its latest rolling count, and the * actual duration of the sliding window. The latter is included in case the expected sliding window length (as * configured by the user) is different from the actual length, e.g. due to high system load. Note that the actual * window length is tracked and calculated for the window, and not individually for each object within a window. - * + *

* Note: During the startup phase you will usually observe that the bolt warns you about the actual sliding window * length being smaller than the expected length. This behavior is expected and is caused by the way the sliding window * counts are initially "loaded up". You can safely ignore this warning during startup (e.g. you will see this warning * during the first ~ five minutes of startup time if the window length is set to five minutes). - * */ public class RollingCountBolt extends BaseRichBolt { - private static final long serialVersionUID = 5537727428628598519L; - private static final Logger LOG = Logger.getLogger(RollingCountBolt.class); - private static final int NUM_WINDOW_CHUNKS = 5; - private static final int DEFAULT_SLIDING_WINDOW_IN_SECONDS = NUM_WINDOW_CHUNKS * 60; - private static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS = DEFAULT_SLIDING_WINDOW_IN_SECONDS / NUM_WINDOW_CHUNKS; - private static final String WINDOW_LENGTH_WARNING_TEMPLATE = "Actual window length is %d seconds when it should be %d seconds" - + " (you can safely ignore this warning during the startup phase)"; - - private final SlidingWindowCounter counter; - private final int windowLengthInSeconds; - private final int emitFrequencyInSeconds; - private OutputCollector collector; - private NthLastModifiedTimeTracker lastModifiedTracker; - - public RollingCountBolt() { - this(DEFAULT_SLIDING_WINDOW_IN_SECONDS, DEFAULT_EMIT_FREQUENCY_IN_SECONDS); + private static final long serialVersionUID = 5537727428628598519L; + private static final Logger LOG = Logger.getLogger(RollingCountBolt.class); + private static final int NUM_WINDOW_CHUNKS = 5; + private static final int DEFAULT_SLIDING_WINDOW_IN_SECONDS = NUM_WINDOW_CHUNKS * 60; + private static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS = DEFAULT_SLIDING_WINDOW_IN_SECONDS / NUM_WINDOW_CHUNKS; + private static final String WINDOW_LENGTH_WARNING_TEMPLATE = + "Actual window length is %d seconds when it should be %d seconds" + + " (you can safely ignore this warning during the startup phase)"; + + private final SlidingWindowCounter counter; + private final int windowLengthInSeconds; + private final int emitFrequencyInSeconds; + private OutputCollector collector; + private NthLastModifiedTimeTracker lastModifiedTracker; + + public RollingCountBolt() { + this(DEFAULT_SLIDING_WINDOW_IN_SECONDS, DEFAULT_EMIT_FREQUENCY_IN_SECONDS); + } + + public RollingCountBolt(int windowLengthInSeconds, int emitFrequencyInSeconds) { + this.windowLengthInSeconds = windowLengthInSeconds; + this.emitFrequencyInSeconds = emitFrequencyInSeconds; + counter = new SlidingWindowCounter(deriveNumWindowChunksFrom(this.windowLengthInSeconds, + this.emitFrequencyInSeconds)); + } + + private int deriveNumWindowChunksFrom(int windowLengthInSeconds, int windowUpdateFrequencyInSeconds) { + return windowLengthInSeconds / windowUpdateFrequencyInSeconds; + } + + @SuppressWarnings("rawtypes") + @Override + public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { + this.collector = collector; + lastModifiedTracker = new NthLastModifiedTimeTracker(deriveNumWindowChunksFrom(this.windowLengthInSeconds, + this.emitFrequencyInSeconds)); + } + + @Override + public void execute(Tuple tuple) { + if (TupleHelpers.isTickTuple(tuple)) { + LOG.debug("Received tick tuple, triggering emit of current window counts"); + emitCurrentWindowCounts(); } - - public RollingCountBolt(int windowLengthInSeconds, int emitFrequencyInSeconds) { - this.windowLengthInSeconds = windowLengthInSeconds; - this.emitFrequencyInSeconds = emitFrequencyInSeconds; - counter = new SlidingWindowCounter(deriveNumWindowChunksFrom(this.windowLengthInSeconds, - this.emitFrequencyInSeconds)); + else { + countObjAndAck(tuple); } - - private int deriveNumWindowChunksFrom(int windowLengthInSeconds, int windowUpdateFrequencyInSeconds) { - return windowLengthInSeconds / windowUpdateFrequencyInSeconds; + } + + private void emitCurrentWindowCounts() { + Map counts = counter.getCountsThenAdvanceWindow(); + int actualWindowLengthInSeconds = lastModifiedTracker.secondsSinceOldestModification(); + lastModifiedTracker.markAsModified(); + if (actualWindowLengthInSeconds != windowLengthInSeconds) { + LOG.warn(String.format(WINDOW_LENGTH_WARNING_TEMPLATE, actualWindowLengthInSeconds, windowLengthInSeconds)); } - - @SuppressWarnings("rawtypes") - @Override - public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { - this.collector = collector; - lastModifiedTracker = new NthLastModifiedTimeTracker(deriveNumWindowChunksFrom(this.windowLengthInSeconds, - this.emitFrequencyInSeconds)); - } - - @Override - public void execute(Tuple tuple) { - if (TupleHelpers.isTickTuple(tuple)) { - LOG.debug("Received tick tuple, triggering emit of current window counts"); - emitCurrentWindowCounts(); - } - else { - countObjAndAck(tuple); - } - } - - private void emitCurrentWindowCounts() { - Map counts = counter.getCountsThenAdvanceWindow(); - int actualWindowLengthInSeconds = lastModifiedTracker.secondsSinceOldestModification(); - lastModifiedTracker.markAsModified(); - if (actualWindowLengthInSeconds != windowLengthInSeconds) { - LOG.warn(String.format(WINDOW_LENGTH_WARNING_TEMPLATE, actualWindowLengthInSeconds, windowLengthInSeconds)); - } - emit(counts, actualWindowLengthInSeconds); - } - - private void emit(Map counts, int actualWindowLengthInSeconds) { - for (Entry entry : counts.entrySet()) { - Object obj = entry.getKey(); - Long count = entry.getValue(); - collector.emit(new Values(obj, count, actualWindowLengthInSeconds)); - } - } - - private void countObjAndAck(Tuple tuple) { - Object obj = tuple.getValue(0); - counter.incrementCount(obj); - collector.ack(tuple); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("obj", "count", "actualWindowLengthInSeconds")); - } - - @Override - public Map getComponentConfiguration() { - Map conf = new HashMap(); - conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, emitFrequencyInSeconds); - return conf; + emit(counts, actualWindowLengthInSeconds); + } + + private void emit(Map counts, int actualWindowLengthInSeconds) { + for (Entry entry : counts.entrySet()) { + Object obj = entry.getKey(); + Long count = entry.getValue(); + collector.emit(new Values(obj, count, actualWindowLengthInSeconds)); } + } + + private void countObjAndAck(Tuple tuple) { + Object obj = tuple.getValue(0); + counter.incrementCount(obj); + collector.ack(tuple); + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("obj", "count", "actualWindowLengthInSeconds")); + } + + @Override + public Map getComponentConfiguration() { + Map conf = new HashMap(); + conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, emitFrequencyInSeconds); + return conf; + } } diff --git a/src/jvm/storm/starter/bolt/SingleJoinBolt.java b/src/jvm/storm/starter/bolt/SingleJoinBolt.java index c353b7a2e2..e12f005a4b 100644 --- a/src/jvm/storm/starter/bolt/SingleJoinBolt.java +++ b/src/jvm/storm/starter/bolt/SingleJoinBolt.java @@ -9,90 +9,89 @@ import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.utils.TimeCacheMap; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; + +import java.util.*; public class SingleJoinBolt extends BaseRichBolt { - OutputCollector _collector; - Fields _idFields; - Fields _outFields; - int _numSources; - TimeCacheMap, Map> _pending; - Map _fieldLocations; - - public SingleJoinBolt(Fields outFields) { - _outFields = outFields; - } - - @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { - _fieldLocations = new HashMap(); - _collector = collector; - int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue(); - _pending = new TimeCacheMap, Map>(timeout, new ExpireCallback()); - _numSources = context.getThisSources().size(); - Set idFields = null; - for(GlobalStreamId source: context.getThisSources().keySet()) { - Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId()); - Set setFields = new HashSet(fields.toList()); - if(idFields==null) idFields = setFields; - else idFields.retainAll(setFields); - - for(String outfield: _outFields) { - for(String sourcefield: fields) { - if(outfield.equals(sourcefield)) { - _fieldLocations.put(outfield, source); - } - } - } - } - _idFields = new Fields(new ArrayList(idFields)); - - if(_fieldLocations.size()!=_outFields.size()) { - throw new RuntimeException("Cannot find all outfields among sources"); + OutputCollector _collector; + Fields _idFields; + Fields _outFields; + int _numSources; + TimeCacheMap, Map> _pending; + Map _fieldLocations; + + public SingleJoinBolt(Fields outFields) { + _outFields = outFields; + } + + @Override + public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + _fieldLocations = new HashMap(); + _collector = collector; + int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue(); + _pending = new TimeCacheMap, Map>(timeout, new ExpireCallback()); + _numSources = context.getThisSources().size(); + Set idFields = null; + for (GlobalStreamId source : context.getThisSources().keySet()) { + Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId()); + Set setFields = new HashSet(fields.toList()); + if (idFields == null) + idFields = setFields; + else + idFields.retainAll(setFields); + + for (String outfield : _outFields) { + for (String sourcefield : fields) { + if (outfield.equals(sourcefield)) { + _fieldLocations.put(outfield, source); + } } + } } + _idFields = new Fields(new ArrayList(idFields)); - @Override - public void execute(Tuple tuple) { - List id = tuple.select(_idFields); - GlobalStreamId streamId = new GlobalStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId()); - if(!_pending.containsKey(id)) { - _pending.put(id, new HashMap()); - } - Map parts = _pending.get(id); - if(parts.containsKey(streamId)) throw new RuntimeException("Received same side of single join twice"); - parts.put(streamId, tuple); - if(parts.size()==_numSources) { - _pending.remove(id); - List joinResult = new ArrayList(); - for(String outField: _outFields) { - GlobalStreamId loc = _fieldLocations.get(outField); - joinResult.add(parts.get(loc).getValueByField(outField)); - } - _collector.emit(new ArrayList(parts.values()), joinResult); - - for(Tuple part: parts.values()) { - _collector.ack(part); - } - } + if (_fieldLocations.size() != _outFields.size()) { + throw new RuntimeException("Cannot find all outfields among sources"); + } + } + + @Override + public void execute(Tuple tuple) { + List id = tuple.select(_idFields); + GlobalStreamId streamId = new GlobalStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId()); + if (!_pending.containsKey(id)) { + _pending.put(id, new HashMap()); } + Map parts = _pending.get(id); + if (parts.containsKey(streamId)) + throw new RuntimeException("Received same side of single join twice"); + parts.put(streamId, tuple); + if (parts.size() == _numSources) { + _pending.remove(id); + List joinResult = new ArrayList(); + for (String outField : _outFields) { + GlobalStreamId loc = _fieldLocations.get(outField); + joinResult.add(parts.get(loc).getValueByField(outField)); + } + _collector.emit(new ArrayList(parts.values()), joinResult); + + for (Tuple part : parts.values()) { + _collector.ack(part); + } + } + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(_outFields); + } + private class ExpireCallback implements TimeCacheMap.ExpiredCallback, Map> { @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(_outFields); - } - - private class ExpireCallback implements TimeCacheMap.ExpiredCallback, Map> { - @Override - public void expire(List id, Map tuples) { - for(Tuple tuple: tuples.values()) { - _collector.fail(tuple); - } - } + public void expire(List id, Map tuples) { + for (Tuple tuple : tuples.values()) { + _collector.fail(tuple); + } } + } } diff --git a/src/jvm/storm/starter/bolt/TotalRankingsBolt.java b/src/jvm/storm/starter/bolt/TotalRankingsBolt.java index 84ff69e5d6..846ff493e5 100644 --- a/src/jvm/storm/starter/bolt/TotalRankingsBolt.java +++ b/src/jvm/storm/starter/bolt/TotalRankingsBolt.java @@ -1,43 +1,41 @@ package storm.starter.bolt; +import backtype.storm.tuple.Tuple; import org.apache.log4j.Logger; - import storm.starter.tools.Rankings; -import backtype.storm.tuple.Tuple; /** * This bolt merges incoming {@link Rankings}. - * + *

* It can be used to merge intermediate rankings generated by {@link IntermediateRankingsBolt} into a final, * consolidated ranking. To do so, configure this bolt with a globalGrouping on {@link IntermediateRankingsBolt}. - * */ public final class TotalRankingsBolt extends AbstractRankerBolt { - private static final long serialVersionUID = -8447525895532302198L; - private static final Logger LOG = Logger.getLogger(TotalRankingsBolt.class); + private static final long serialVersionUID = -8447525895532302198L; + private static final Logger LOG = Logger.getLogger(TotalRankingsBolt.class); - public TotalRankingsBolt() { - super(); - } + public TotalRankingsBolt() { + super(); + } - public TotalRankingsBolt(int topN) { - super(topN); - } + public TotalRankingsBolt(int topN) { + super(topN); + } - public TotalRankingsBolt(int topN, int emitFrequencyInSeconds) { - super(topN, emitFrequencyInSeconds); - } + public TotalRankingsBolt(int topN, int emitFrequencyInSeconds) { + super(topN, emitFrequencyInSeconds); + } - @Override - void updateRankingsWithTuple(Tuple tuple) { - Rankings rankingsToBeMerged = (Rankings) tuple.getValue(0); - super.getRankings().updateWith(rankingsToBeMerged); - } + @Override + void updateRankingsWithTuple(Tuple tuple) { + Rankings rankingsToBeMerged = (Rankings) tuple.getValue(0); + super.getRankings().updateWith(rankingsToBeMerged); + } - @Override - Logger getLogger() { - return LOG; - } + @Override + Logger getLogger() { + return LOG; + } } diff --git a/src/jvm/storm/starter/spout/RandomSentenceSpout.java b/src/jvm/storm/starter/spout/RandomSentenceSpout.java index af6c151d8d..e808f8361c 100644 --- a/src/jvm/storm/starter/spout/RandomSentenceSpout.java +++ b/src/jvm/storm/starter/spout/RandomSentenceSpout.java @@ -7,44 +7,41 @@ import backtype.storm.tuple.Fields; import backtype.storm.tuple.Values; import backtype.storm.utils.Utils; + import java.util.Map; import java.util.Random; public class RandomSentenceSpout extends BaseRichSpout { - SpoutOutputCollector _collector; - Random _rand; - - - @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { - _collector = collector; - _rand = new Random(); - } - - @Override - public void nextTuple() { - Utils.sleep(100); - String[] sentences = new String[] { - "the cow jumped over the moon", - "an apple a day keeps the doctor away", - "four score and seven years ago", - "snow white and the seven dwarfs", - "i am at two with nature"}; - String sentence = sentences[_rand.nextInt(sentences.length)]; - _collector.emit(new Values(sentence)); - } - - @Override - public void ack(Object id) { - } - - @Override - public void fail(Object id) { - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("word")); - } - + SpoutOutputCollector _collector; + Random _rand; + + + @Override + public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + _collector = collector; + _rand = new Random(); + } + + @Override + public void nextTuple() { + Utils.sleep(100); + String[] sentences = new String[]{ "the cow jumped over the moon", "an apple a day keeps the doctor away", + "four score and seven years ago", "snow white and the seven dwarfs", "i am at two with nature" }; + String sentence = sentences[_rand.nextInt(sentences.length)]; + _collector.emit(new Values(sentence)); + } + + @Override + public void ack(Object id) { + } + + @Override + public void fail(Object id) { + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("word")); + } + } \ No newline at end of file diff --git a/src/jvm/storm/starter/tools/NthLastModifiedTimeTracker.java b/src/jvm/storm/starter/tools/NthLastModifiedTimeTracker.java index 4b85cd27f8..92998a6d3e 100644 --- a/src/jvm/storm/starter/tools/NthLastModifiedTimeTracker.java +++ b/src/jvm/storm/starter/tools/NthLastModifiedTimeTracker.java @@ -1,55 +1,53 @@ package storm.starter.tools; -import org.apache.commons.collections.buffer.CircularFifoBuffer; - import backtype.storm.utils.Time; +import org.apache.commons.collections.buffer.CircularFifoBuffer; /** * This class tracks the time-since-last-modify of a "thing" in a rolling fashion. - * + *

* For example, create a 5-slot tracker to track the five most recent time-since-last-modify. - * + *

* You must manually "mark" that the "something" that you want to track -- in terms of modification times -- has just * been modified. - * */ public class NthLastModifiedTimeTracker { - private static final int MILLIS_IN_SEC = 1000; + private static final int MILLIS_IN_SEC = 1000; - private final CircularFifoBuffer lastModifiedTimesMillis; + private final CircularFifoBuffer lastModifiedTimesMillis; - public NthLastModifiedTimeTracker(int numTimesToTrack) { - if (numTimesToTrack < 1) { - throw new IllegalArgumentException("numTimesToTrack must be greater than zero (you requested " - + numTimesToTrack + ")"); - } - lastModifiedTimesMillis = new CircularFifoBuffer(numTimesToTrack); - initLastModifiedTimesMillis(); + public NthLastModifiedTimeTracker(int numTimesToTrack) { + if (numTimesToTrack < 1) { + throw new IllegalArgumentException( + "numTimesToTrack must be greater than zero (you requested " + numTimesToTrack + ")"); } - - private void initLastModifiedTimesMillis() { - long nowCached = now(); - for (int i = 0; i < lastModifiedTimesMillis.maxSize(); i++) { - lastModifiedTimesMillis.add(Long.valueOf(nowCached)); - } + lastModifiedTimesMillis = new CircularFifoBuffer(numTimesToTrack); + initLastModifiedTimesMillis(); + } + + private void initLastModifiedTimesMillis() { + long nowCached = now(); + for (int i = 0; i < lastModifiedTimesMillis.maxSize(); i++) { + lastModifiedTimesMillis.add(Long.valueOf(nowCached)); } + } - private long now() { - return Time.currentTimeMillis(); - } + private long now() { + return Time.currentTimeMillis(); + } - public int secondsSinceOldestModification() { - long modifiedTimeMillis = ((Long) lastModifiedTimesMillis.get()).longValue(); - return (int) ((now() - modifiedTimeMillis) / MILLIS_IN_SEC); - } + public int secondsSinceOldestModification() { + long modifiedTimeMillis = ((Long) lastModifiedTimesMillis.get()).longValue(); + return (int) ((now() - modifiedTimeMillis) / MILLIS_IN_SEC); + } - public void markAsModified() { - updateLastModifiedTime(); - } + public void markAsModified() { + updateLastModifiedTime(); + } - private void updateLastModifiedTime() { - lastModifiedTimesMillis.add(now()); - } + private void updateLastModifiedTime() { + lastModifiedTimesMillis.add(now()); + } } diff --git a/src/jvm/storm/starter/tools/Rankable.java b/src/jvm/storm/starter/tools/Rankable.java index 03f742fa03..36ba0860d9 100644 --- a/src/jvm/storm/starter/tools/Rankable.java +++ b/src/jvm/storm/starter/tools/Rankable.java @@ -2,8 +2,8 @@ public interface Rankable extends Comparable { - Object getObject(); + Object getObject(); - long getCount(); + long getCount(); } diff --git a/src/jvm/storm/starter/tools/RankableObjectWithFields.java b/src/jvm/storm/starter/tools/RankableObjectWithFields.java index 38db6e9231..1fdad374ce 100644 --- a/src/jvm/storm/starter/tools/RankableObjectWithFields.java +++ b/src/jvm/storm/starter/tools/RankableObjectWithFields.java @@ -1,119 +1,118 @@ package storm.starter.tools; -import java.io.Serializable; -import java.util.List; - import backtype.storm.tuple.Tuple; - import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import java.io.Serializable; +import java.util.List; + /** * This class wraps an objects and its associated count, including any additional data fields. - * + *

* This class can be used, for instance, to track the number of occurrences of an object in a Storm topology. - * */ public class RankableObjectWithFields implements Rankable, Serializable { - private static final long serialVersionUID = -9102878650001058090L; - private static final String toStringSeparator = "|"; - - private final Object obj; - private final long count; - private final ImmutableList fields; + private static final long serialVersionUID = -9102878650001058090L; + private static final String toStringSeparator = "|"; - public RankableObjectWithFields(Object obj, long count, Object... otherFields) { - if (obj == null) { - throw new IllegalArgumentException("The object must not be null"); - } - if (count < 0) { - throw new IllegalArgumentException("The count must be >= 0"); - } - this.obj = obj; - this.count = count; - fields = ImmutableList.copyOf(otherFields); + private final Object obj; + private final long count; + private final ImmutableList fields; + public RankableObjectWithFields(Object obj, long count, Object... otherFields) { + if (obj == null) { + throw new IllegalArgumentException("The object must not be null"); } - - /** - * Construct a new instance based on the provided {@link Tuple}. - * - * This method expects the object to be ranked in the first field (index 0) of the provided tuple, and the number of - * occurrences of the object (its count) in the second field (index 1). Any further fields in the tuple will be - * extracted and tracked, too. These fields can be accessed via {@link RankableObjectWithFields#getFields()}. - * - * @param tuple - * @return - */ - public static RankableObjectWithFields from(Tuple tuple) { - List otherFields = Lists.newArrayList(tuple.getValues()); - Object obj = otherFields.remove(0); - Long count = (Long) otherFields.remove(0); - return new RankableObjectWithFields(obj, count, otherFields.toArray()); - } - - public Object getObject() { - return obj; + if (count < 0) { + throw new IllegalArgumentException("The count must be >= 0"); } - - public long getCount() { - return count; + this.obj = obj; + this.count = count; + fields = ImmutableList.copyOf(otherFields); + + } + + /** + * Construct a new instance based on the provided {@link Tuple}. + *

+ * This method expects the object to be ranked in the first field (index 0) of the provided tuple, and the number of + * occurrences of the object (its count) in the second field (index 1). Any further fields in the tuple will be + * extracted and tracked, too. These fields can be accessed via {@link RankableObjectWithFields#getFields()}. + * + * @param tuple + * + * @return + */ + public static RankableObjectWithFields from(Tuple tuple) { + List otherFields = Lists.newArrayList(tuple.getValues()); + Object obj = otherFields.remove(0); + Long count = (Long) otherFields.remove(0); + return new RankableObjectWithFields(obj, count, otherFields.toArray()); + } + + public Object getObject() { + return obj; + } + + public long getCount() { + return count; + } + + /** + * @return an immutable list of any additional data fields of the object (may be empty but will never be null) + */ + public List getFields() { + return fields; + } + + @Override + public int compareTo(Rankable other) { + long delta = this.getCount() - other.getCount(); + if (delta > 0) { + return 1; } - - /** - * @return an immutable list of any additional data fields of the object (may be empty but will never be null) - */ - public List getFields() { - return fields; + else if (delta < 0) { + return -1; } - - @Override - public int compareTo(Rankable other) { - long delta = this.getCount() - other.getCount(); - if (delta > 0) { - return 1; - } - else if (delta < 0) { - return -1; - } - else { - return 0; - } + else { + return 0; } + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof RankableObjectWithFields)) { - return false; - } - RankableObjectWithFields other = (RankableObjectWithFields) o; - return obj.equals(other.obj) && count == other.count; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - int result = 17; - int countHash = (int) (count ^ (count >>> 32)); - result = 31 * result + countHash; - result = 31 * result + obj.hashCode(); - return result; + if (!(o instanceof RankableObjectWithFields)) { + return false; } - - public String toString() { - StringBuffer buf = new StringBuffer(); - buf.append("["); - buf.append(obj); - buf.append(toStringSeparator); - buf.append(count); - for (Object field : fields) { - buf.append(toStringSeparator); - buf.append(field); - } - buf.append("]"); - return buf.toString(); + RankableObjectWithFields other = (RankableObjectWithFields) o; + return obj.equals(other.obj) && count == other.count; + } + + @Override + public int hashCode() { + int result = 17; + int countHash = (int) (count ^ (count >>> 32)); + result = 31 * result + countHash; + result = 31 * result + obj.hashCode(); + return result; + } + + public String toString() { + StringBuffer buf = new StringBuffer(); + buf.append("["); + buf.append(obj); + buf.append(toStringSeparator); + buf.append(count); + for (Object field : fields) { + buf.append(toStringSeparator); + buf.append(field); } + buf.append("]"); + return buf.toString(); + } } diff --git a/src/jvm/storm/starter/tools/Rankings.java b/src/jvm/storm/starter/tools/Rankings.java index d6fd34ca96..a3f421a5fe 100644 --- a/src/jvm/storm/starter/tools/Rankings.java +++ b/src/jvm/storm/starter/tools/Rankings.java @@ -1,95 +1,95 @@ package storm.starter.tools; +import com.google.common.collect.Lists; + import java.io.Serializable; import java.util.Collections; import java.util.List; -import com.google.common.collect.Lists; - public class Rankings implements Serializable { - private static final long serialVersionUID = -1549827195410578903L; - private static final int DEFAULT_COUNT = 10; - - private final int maxSize; - private final List rankedItems = Lists.newArrayList(); - - public Rankings() { - this(DEFAULT_COUNT); - } + private static final long serialVersionUID = -1549827195410578903L; + private static final int DEFAULT_COUNT = 10; - public Rankings(int topN) { - if (topN < 1) { - throw new IllegalArgumentException("topN must be >= 1"); - } - maxSize = topN; - } + private final int maxSize; + private final List rankedItems = Lists.newArrayList(); - /** - * @return the maximum possible number (size) of ranked objects this instance can hold - */ - public int maxSize() { - return maxSize; - } + public Rankings() { + this(DEFAULT_COUNT); + } - /** - * @return the number (size) of ranked objects this instance is currently holding - */ - public int size() { - return rankedItems.size(); + public Rankings(int topN) { + if (topN < 1) { + throw new IllegalArgumentException("topN must be >= 1"); } - - public List getRankings() { - return Lists.newArrayList(rankedItems); + maxSize = topN; + } + + /** + * @return the maximum possible number (size) of ranked objects this instance can hold + */ + public int maxSize() { + return maxSize; + } + + /** + * @return the number (size) of ranked objects this instance is currently holding + */ + public int size() { + return rankedItems.size(); + } + + public List getRankings() { + return Lists.newArrayList(rankedItems); + } + + public void updateWith(Rankings other) { + for (Rankable r : other.getRankings()) { + updateWith(r); } + } - public void updateWith(Rankings other) { - for (Rankable r : other.getRankings()) { - updateWith(r); - } + public void updateWith(Rankable r) { + synchronized(rankedItems) { + addOrReplace(r); + rerank(); + shrinkRankingsIfNeeded(); } + } - public void updateWith(Rankable r) { - synchronized(rankedItems) { - addOrReplace(r); - rerank(); - shrinkRankingsIfNeeded(); - } + private void addOrReplace(Rankable r) { + Integer rank = findRankOf(r); + if (rank != null) { + rankedItems.set(rank, r); } - - private void addOrReplace(Rankable r) { - Integer rank = findRankOf(r); - if (rank != null) { - rankedItems.set(rank, r); - } - else { - rankedItems.add(r); - } + else { + rankedItems.add(r); } - - private Integer findRankOf(Rankable r) { - Object tag = r.getObject(); - for (int rank = 0; rank < rankedItems.size(); rank++) { - Object cur = rankedItems.get(rank).getObject(); - if (cur.equals(tag)) { - return rank; - } - } - return null; + } + + private Integer findRankOf(Rankable r) { + Object tag = r.getObject(); + for (int rank = 0; rank < rankedItems.size(); rank++) { + Object cur = rankedItems.get(rank).getObject(); + if (cur.equals(tag)) { + return rank; + } } + return null; + } - private void rerank() { - Collections.sort(rankedItems); - Collections.reverse(rankedItems); - } + private void rerank() { + Collections.sort(rankedItems); + Collections.reverse(rankedItems); + } - private void shrinkRankingsIfNeeded() { - if (rankedItems.size() > maxSize) { - rankedItems.remove(maxSize); - } + private void shrinkRankingsIfNeeded() { + if (rankedItems.size() > maxSize) { + rankedItems.remove(maxSize); } + } - public String toString() { - return rankedItems.toString(); - } + public String toString() { + return rankedItems.toString(); + } } diff --git a/src/jvm/storm/starter/tools/SlidingWindowCounter.java b/src/jvm/storm/starter/tools/SlidingWindowCounter.java index b20af00803..dff6a593f8 100644 --- a/src/jvm/storm/starter/tools/SlidingWindowCounter.java +++ b/src/jvm/storm/starter/tools/SlidingWindowCounter.java @@ -5,99 +5,98 @@ /** * This class counts objects in a sliding window fashion. - * + *

* It is designed 1) to give multiple "producer" threads write access to the counter, i.e. being able to increment * counts of objects, and 2) to give a single "consumer" thread (e.g. {@link PeriodicSlidingWindowCounter}) read access * to the counter. Whenever the consumer thread performs a read operation, this class will advance the head slot of the * sliding window counter. This means that the consumer thread indirectly controls where writes of the producer threads * will go to. Also, by itself this class will not advance the head slot. - * + *

* A note for analyzing data based on a sliding window count: During the initial windowLengthInSlots * iterations, this sliding window counter will always return object counts that are equal or greater than in the * previous iteration. This is the effect of the counter "loading up" at the very start of its existence. Conceptually, * this is the desired behavior. - * + *

* To give an example, using a counter with 5 slots which for the sake of this example represent 1 minute of time each: - * + *

* 

  * {@code
  * Sliding window counts of an object X over time
- * 
+ *
  * Minute (timeline):
  * 1    2   3   4   5   6   7   8
- * 
+ *
  * Observed counts per minute:
  * 1    1   1   1   0   0   0   0
- * 
+ *
  * Counts returned by counter:
  * 1    2   3   4   4   3   2   1
  * }
  *
- * + *

* As you can see in this example, for the first windowLengthInSlots (here: the first five minutes) the * counter will always return counts equal or greater than in the previous iteration (1, 2, 3, 4, 4). This initial load * effect needs to be accounted for whenever you want to perform analyses such as trending topics; otherwise your * analysis algorithm might falsely identify the object to be trending as the counter seems to observe continuously * increasing counts. Also, note that during the initial load phase every object will exhibit increasing * counts. - * + *

* On a high-level, the counter exhibits the following behavior: If you asked the example counter after two minutes, - * "how often did you count the object during the past five minutes?", then it should reply - * "I have counted it 2 times in the past five minutes", implying that it can only account for the last two of those - * five minutes because the counter was not running before that time. - * - * @param - * The type of those objects we want to count. + * "how often did you count the object during the past five minutes?", then it should reply "I have counted it 2 times + * in the past five minutes", implying that it can only account for the last two of those five minutes because the + * counter was not running before that time. + * + * @param The type of those objects we want to count. */ public final class SlidingWindowCounter implements Serializable { - private static final long serialVersionUID = -2645063988768785810L; + private static final long serialVersionUID = -2645063988768785810L; - private SlotBasedCounter objCounter; - private int headSlot; - private int tailSlot; - private int windowLengthInSlots; + private SlotBasedCounter objCounter; + private int headSlot; + private int tailSlot; + private int windowLengthInSlots; - public SlidingWindowCounter(int windowLengthInSlots) { - if (windowLengthInSlots < 2) { - throw new IllegalArgumentException("Window length in slots must be at least two (you requested " - + windowLengthInSlots + ")"); - } - this.windowLengthInSlots = windowLengthInSlots; - this.objCounter = new SlotBasedCounter(this.windowLengthInSlots); - - this.headSlot = 0; - this.tailSlot = slotAfter(headSlot); + public SlidingWindowCounter(int windowLengthInSlots) { + if (windowLengthInSlots < 2) { + throw new IllegalArgumentException( + "Window length in slots must be at least two (you requested " + windowLengthInSlots + ")"); } + this.windowLengthInSlots = windowLengthInSlots; + this.objCounter = new SlotBasedCounter(this.windowLengthInSlots); - public void incrementCount(T obj) { - objCounter.incrementCount(obj, headSlot); - } + this.headSlot = 0; + this.tailSlot = slotAfter(headSlot); + } - /** - * Return the current (total) counts of all tracked objects, then advance the window. - * - * Whenever this method is called, we consider the counts of the current sliding window to be available to and - * successfully processed "upstream" (i.e. by the caller). Knowing this we will start counting any subsequent - * objects within the next "chunk" of the sliding window. - * - * @return - */ - public Map getCountsThenAdvanceWindow() { - Map counts = objCounter.getCounts(); - objCounter.wipeZeros(); - objCounter.wipeSlot(tailSlot); - advanceHead(); - return counts; - } + public void incrementCount(T obj) { + objCounter.incrementCount(obj, headSlot); + } - private void advanceHead() { - headSlot = tailSlot; - tailSlot = slotAfter(tailSlot); - } + /** + * Return the current (total) counts of all tracked objects, then advance the window. + *

+ * Whenever this method is called, we consider the counts of the current sliding window to be available to and + * successfully processed "upstream" (i.e. by the caller). Knowing this we will start counting any subsequent + * objects within the next "chunk" of the sliding window. + * + * @return + */ + public Map getCountsThenAdvanceWindow() { + Map counts = objCounter.getCounts(); + objCounter.wipeZeros(); + objCounter.wipeSlot(tailSlot); + advanceHead(); + return counts; + } - private int slotAfter(int slot) { - return (slot + 1) % windowLengthInSlots; - } + private void advanceHead() { + headSlot = tailSlot; + tailSlot = slotAfter(tailSlot); + } + + private int slotAfter(int slot) { + return (slot + 1) % windowLengthInSlots; + } } diff --git a/src/jvm/storm/starter/tools/SlotBasedCounter.java b/src/jvm/storm/starter/tools/SlotBasedCounter.java index 5624a54fc3..f45ef48a1f 100644 --- a/src/jvm/storm/starter/tools/SlotBasedCounter.java +++ b/src/jvm/storm/starter/tools/SlotBasedCounter.java @@ -8,96 +8,94 @@ /** * This class provides per-slot counts of the occurrences of objects. - * + *

* It can be used, for instance, as a building block for implementing sliding window counting of objects. - * - * @param - * The type of those objects we want to count. + * + * @param The type of those objects we want to count. */ public final class SlotBasedCounter implements Serializable { - private static final long serialVersionUID = 4858185737378394432L; + private static final long serialVersionUID = 4858185737378394432L; - private final Map objToCounts = new HashMap(); - private final int numSlots; + private final Map objToCounts = new HashMap(); + private final int numSlots; - public SlotBasedCounter(int numSlots) { - if (numSlots <= 0) { - throw new IllegalArgumentException("Number of slots must be greater than zero (you requested " + numSlots - + ")"); - } - this.numSlots = numSlots; + public SlotBasedCounter(int numSlots) { + if (numSlots <= 0) { + throw new IllegalArgumentException("Number of slots must be greater than zero (you requested " + numSlots + ")"); } + this.numSlots = numSlots; + } - public void incrementCount(T obj, int slot) { - long[] counts = objToCounts.get(obj); - if (counts == null) { - counts = new long[this.numSlots]; - objToCounts.put(obj, counts); - } - counts[slot]++; + public void incrementCount(T obj, int slot) { + long[] counts = objToCounts.get(obj); + if (counts == null) { + counts = new long[this.numSlots]; + objToCounts.put(obj, counts); } + counts[slot]++; + } - public long getCount(T obj, int slot) { - long[] counts = objToCounts.get(obj); - if (counts == null) { - return 0; - } - else { - return counts[slot]; - } + public long getCount(T obj, int slot) { + long[] counts = objToCounts.get(obj); + if (counts == null) { + return 0; } - - public Map getCounts() { - Map result = new HashMap(); - for (T obj : objToCounts.keySet()) { - result.put(obj, computeTotalCount(obj)); - } - return result; + else { + return counts[slot]; } + } - private long computeTotalCount(T obj) { - long[] curr = objToCounts.get(obj); - long total = 0; - for (long l : curr) { - total += l; - } - return total; + public Map getCounts() { + Map result = new HashMap(); + for (T obj : objToCounts.keySet()) { + result.put(obj, computeTotalCount(obj)); } + return result; + } - /** - * Reset the slot count of any tracked objects to zero for the given slot. - * - * @param slot - */ - public void wipeSlot(int slot) { - for (T obj : objToCounts.keySet()) { - resetSlotCountToZero(obj, slot); - } + private long computeTotalCount(T obj) { + long[] curr = objToCounts.get(obj); + long total = 0; + for (long l : curr) { + total += l; } + return total; + } - private void resetSlotCountToZero(T obj, int slot) { - long[] counts = objToCounts.get(obj); - counts[slot] = 0; + /** + * Reset the slot count of any tracked objects to zero for the given slot. + * + * @param slot + */ + public void wipeSlot(int slot) { + for (T obj : objToCounts.keySet()) { + resetSlotCountToZero(obj, slot); } + } - private boolean shouldBeRemovedFromCounter(T obj) { - return computeTotalCount(obj) == 0; - } + private void resetSlotCountToZero(T obj, int slot) { + long[] counts = objToCounts.get(obj); + counts[slot] = 0; + } - /** - * Remove any object from the counter whose total count is zero (to free up memory). - */ - public void wipeZeros() { - Set objToBeRemoved = new HashSet(); - for (T obj : objToCounts.keySet()) { - if (shouldBeRemovedFromCounter(obj)) { - objToBeRemoved.add(obj); - } - } - for (T obj : objToBeRemoved) { - objToCounts.remove(obj); - } + private boolean shouldBeRemovedFromCounter(T obj) { + return computeTotalCount(obj) == 0; + } + + /** + * Remove any object from the counter whose total count is zero (to free up memory). + */ + public void wipeZeros() { + Set objToBeRemoved = new HashSet(); + for (T obj : objToCounts.keySet()) { + if (shouldBeRemovedFromCounter(obj)) { + objToBeRemoved.add(obj); + } + } + for (T obj : objToBeRemoved) { + objToCounts.remove(obj); } + } } diff --git a/src/jvm/storm/starter/trident/TridentReach.java b/src/jvm/storm/starter/trident/TridentReach.java index ec1014895f..a940b40004 100644 --- a/src/jvm/storm/starter/trident/TridentReach.java +++ b/src/jvm/storm/starter/trident/TridentReach.java @@ -3,15 +3,10 @@ import backtype.storm.Config; import backtype.storm.LocalCluster; import backtype.storm.LocalDRPC; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Values; import backtype.storm.generated.StormTopology; import backtype.storm.task.IMetricsContext; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Values; import storm.trident.TridentState; import storm.trident.TridentTopology; import storm.trident.operation.BaseFunction; @@ -25,126 +20,120 @@ import storm.trident.state.map.ReadOnlyMapState; import storm.trident.tuple.TridentTuple; +import java.util.*; + public class TridentReach { - public static Map> TWEETERS_DB = new HashMap>() {{ - put("foo.com/blog/1", Arrays.asList("sally", "bob", "tim", "george", "nathan")); - put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally", "nathan")); - put("tech.backtype.com/blog/123", Arrays.asList("tim", "mike", "john")); - }}; - - public static Map> FOLLOWERS_DB = new HashMap>() {{ - put("sally", Arrays.asList("bob", "tim", "alice", "adam", "jim", "chris", "jai")); - put("bob", Arrays.asList("sally", "nathan", "jim", "mary", "david", "vivian")); - put("tim", Arrays.asList("alex")); - put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily", "jordan")); - put("adam", Arrays.asList("david", "carissa")); - put("mike", Arrays.asList("john", "bob")); - put("john", Arrays.asList("alice", "nathan", "jim", "mike", "bob")); - }}; - - public static class StaticSingleKeyMapState extends ReadOnlyState implements ReadOnlyMapState { - public static class Factory implements StateFactory { - Map _map; - - public Factory(Map map) { - _map = map; - } - - @Override - public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { - return new StaticSingleKeyMapState(_map); - } + public static Map> TWEETERS_DB = new HashMap>() {{ + put("foo.com/blog/1", Arrays.asList("sally", "bob", "tim", "george", "nathan")); + put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally", "nathan")); + put("tech.backtype.com/blog/123", Arrays.asList("tim", "mike", "john")); + }}; + + public static Map> FOLLOWERS_DB = new HashMap>() {{ + put("sally", Arrays.asList("bob", "tim", "alice", "adam", "jim", "chris", "jai")); + put("bob", Arrays.asList("sally", "nathan", "jim", "mary", "david", "vivian")); + put("tim", Arrays.asList("alex")); + put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily", "jordan")); + put("adam", Arrays.asList("david", "carissa")); + put("mike", Arrays.asList("john", "bob")); + put("john", Arrays.asList("alice", "nathan", "jim", "mike", "bob")); + }}; + + public static class StaticSingleKeyMapState extends ReadOnlyState implements ReadOnlyMapState { + public static class Factory implements StateFactory { + Map _map; + + public Factory(Map map) { + _map = map; + } + + @Override + public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { + return new StaticSingleKeyMapState(_map); + } - } + } - Map _map; + Map _map; - public StaticSingleKeyMapState(Map map) { - _map = map; - } + public StaticSingleKeyMapState(Map map) { + _map = map; + } - @Override - public List multiGet(List> keys) { - List ret = new ArrayList(); - for(List key: keys) { - Object singleKey = key.get(0); - ret.add(_map.get(singleKey)); - } - return ret; - } + @Override + public List multiGet(List> keys) { + List ret = new ArrayList(); + for (List key : keys) { + Object singleKey = key.get(0); + ret.add(_map.get(singleKey)); + } + return ret; + } + } + + public static class One implements CombinerAggregator { + @Override + public Integer init(TridentTuple tuple) { + return 1; } - public static class One implements CombinerAggregator { - @Override - public Integer init(TridentTuple tuple) { - return 1; - } + @Override + public Integer combine(Integer val1, Integer val2) { + return 1; + } - @Override - public Integer combine(Integer val1, Integer val2) { - return 1; - } + @Override + public Integer zero() { + return 1; + } + } - @Override - public Integer zero() { - return 1; + public static class ExpandList extends BaseFunction { + + @Override + public void execute(TridentTuple tuple, TridentCollector collector) { + List l = (List) tuple.getValue(0); + if (l != null) { + for (Object o : l) { + collector.emit(new Values(o)); } + } } - public static class ExpandList extends BaseFunction { + } - @Override - public void execute(TridentTuple tuple, TridentCollector collector) { - List l = (List) tuple.getValue(0); - if(l!=null) { - for(Object o: l) { - collector.emit(new Values(o)); - } - } - } + public static StormTopology buildTopology(LocalDRPC drpc) { + TridentTopology topology = new TridentTopology(); + TridentState urlToTweeters = topology.newStaticState(new StaticSingleKeyMapState.Factory(TWEETERS_DB)); + TridentState tweetersToFollowers = topology.newStaticState(new StaticSingleKeyMapState.Factory(FOLLOWERS_DB)); - } - public static StormTopology buildTopology(LocalDRPC drpc) { - TridentTopology topology = new TridentTopology(); - TridentState urlToTweeters = - topology.newStaticState( - new StaticSingleKeyMapState.Factory(TWEETERS_DB)); - TridentState tweetersToFollowers = - topology.newStaticState( - new StaticSingleKeyMapState.Factory(FOLLOWERS_DB)); - - - topology.newDRPCStream("reach", drpc) - .stateQuery(urlToTweeters, new Fields("args"), new MapGet(), new Fields("tweeters")) - .each(new Fields("tweeters"), new ExpandList(), new Fields("tweeter")) - .shuffle() - .stateQuery(tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers")) - .each(new Fields("followers"), new ExpandList(), new Fields("follower")) - .groupBy(new Fields("follower")) - .aggregate(new One(), new Fields("one")) - .aggregate(new Fields("one"), new Sum(), new Fields("reach")); - return topology.build(); - } + topology.newDRPCStream("reach", drpc).stateQuery(urlToTweeters, new Fields("args"), new MapGet(), new Fields( + "tweeters")).each(new Fields("tweeters"), new ExpandList(), new Fields("tweeter")).shuffle().stateQuery( + tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers")).each(new Fields("followers"), + new ExpandList(), new Fields("follower")).groupBy(new Fields("follower")).aggregate(new One(), new Fields( + "one")).aggregate(new Fields("one"), new Sum(), new Fields("reach")); + return topology.build(); + } - public static void main(String[] args) throws Exception { - LocalDRPC drpc = new LocalDRPC(); + public static void main(String[] args) throws Exception { + LocalDRPC drpc = new LocalDRPC(); - Config conf = new Config(); - LocalCluster cluster = new LocalCluster(); + Config conf = new Config(); + LocalCluster cluster = new LocalCluster(); - cluster.submitTopology("reach", conf, buildTopology(drpc)); + cluster.submitTopology("reach", conf, buildTopology(drpc)); - Thread.sleep(2000); + Thread.sleep(2000); - System.out.println("REACH: " + drpc.execute("reach", "aaa")); - System.out.println("REACH: " + drpc.execute("reach", "foo.com/blog/1")); - System.out.println("REACH: " + drpc.execute("reach", "engineering.twitter.com/blog/5")); + System.out.println("REACH: " + drpc.execute("reach", "aaa")); + System.out.println("REACH: " + drpc.execute("reach", "foo.com/blog/1")); + System.out.println("REACH: " + drpc.execute("reach", "engineering.twitter.com/blog/5")); - cluster.shutdown(); - drpc.shutdown(); - } + cluster.shutdown(); + drpc.shutdown(); + } } diff --git a/src/jvm/storm/starter/trident/TridentWordCount.java b/src/jvm/storm/starter/trident/TridentWordCount.java index 268d41cb19..c56280c59b 100644 --- a/src/jvm/storm/starter/trident/TridentWordCount.java +++ b/src/jvm/storm/starter/trident/TridentWordCount.java @@ -1,6 +1,5 @@ package storm.starter.trident; -import storm.trident.testing.FixedBatchSpout; import backtype.storm.Config; import backtype.storm.LocalCluster; import backtype.storm.LocalDRPC; @@ -16,65 +15,54 @@ import storm.trident.operation.builtin.FilterNull; import storm.trident.operation.builtin.MapGet; import storm.trident.operation.builtin.Sum; -import storm.trident.planner.processor.StateQueryProcessor; +import storm.trident.testing.FixedBatchSpout; import storm.trident.testing.MemoryMapState; import storm.trident.tuple.TridentTuple; -public class TridentWordCount { - public static class Split extends BaseFunction { - @Override - public void execute(TridentTuple tuple, TridentCollector collector) { - String sentence = tuple.getString(0); - for(String word: sentence.split(" ")) { - collector.emit(new Values(word)); - } - } +public class TridentWordCount { + public static class Split extends BaseFunction { + @Override + public void execute(TridentTuple tuple, TridentCollector collector) { + String sentence = tuple.getString(0); + for (String word : sentence.split(" ")) { + collector.emit(new Values(word)); + } } - - public static StormTopology buildTopology(LocalDRPC drpc) { - FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, - new Values("the cow jumped over the moon"), - new Values("the man went to the store and bought some candy"), - new Values("four score and seven years ago"), - new Values("how many apples can you eat"), - new Values("to be or not to be the person")); - spout.setCycle(true); - - TridentTopology topology = new TridentTopology(); - TridentState wordCounts = - topology.newStream("spout1", spout) - .parallelismHint(16) - .each(new Fields("sentence"), new Split(), new Fields("word")) - .groupBy(new Fields("word")) - .persistentAggregate(new MemoryMapState.Factory(), - new Count(), new Fields("count")) - .parallelismHint(16); - - topology.newDRPCStream("words", drpc) - .each(new Fields("args"), new Split(), new Fields("word")) - .groupBy(new Fields("word")) - .stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count")) - .each(new Fields("count"), new FilterNull()) - .aggregate(new Fields("count"), new Sum(), new Fields("sum")) - ; - return topology.build(); + } + + public static StormTopology buildTopology(LocalDRPC drpc) { + FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"), + new Values("the man went to the store and bought some candy"), new Values("four score and seven years ago"), + new Values("how many apples can you eat"), new Values("to be or not to be the person")); + spout.setCycle(true); + + TridentTopology topology = new TridentTopology(); + TridentState wordCounts = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"), + new Split(), new Fields("word")).groupBy(new Fields("word")).persistentAggregate(new MemoryMapState.Factory(), + new Count(), new Fields("count")).parallelismHint(16); + + topology.newDRPCStream("words", drpc).each(new Fields("args"), new Split(), new Fields("word")).groupBy(new Fields( + "word")).stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count")).each(new Fields("count"), + new FilterNull()).aggregate(new Fields("count"), new Sum(), new Fields("sum")); + return topology.build(); + } + + public static void main(String[] args) throws Exception { + Config conf = new Config(); + conf.setMaxSpoutPending(20); + if (args.length == 0) { + LocalDRPC drpc = new LocalDRPC(); + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology("wordCounter", conf, buildTopology(drpc)); + for (int i = 0; i < 100; i++) { + System.out.println("DRPC RESULT: " + drpc.execute("words", "cat the dog jumped")); + Thread.sleep(1000); + } } - - public static void main(String[] args) throws Exception { - Config conf = new Config(); - conf.setMaxSpoutPending(20); - if(args.length==0) { - LocalDRPC drpc = new LocalDRPC(); - LocalCluster cluster = new LocalCluster(); - cluster.submitTopology("wordCounter", conf, buildTopology(drpc)); - for(int i=0; i<100; i++) { - System.out.println("DRPC RESULT: " + drpc.execute("words", "cat the dog jumped")); - Thread.sleep(1000); - } - } else { - conf.setNumWorkers(3); - StormSubmitter.submitTopology(args[0], conf, buildTopology(null)); - } + else { + conf.setNumWorkers(3); + StormSubmitter.submitTopology(args[0], conf, buildTopology(null)); } + } } diff --git a/src/jvm/storm/starter/util/StormRunner.java b/src/jvm/storm/starter/util/StormRunner.java index 09b40628cc..30c4f50d2c 100644 --- a/src/jvm/storm/starter/util/StormRunner.java +++ b/src/jvm/storm/starter/util/StormRunner.java @@ -6,17 +6,17 @@ public final class StormRunner { - private static final int MILLIS_IN_SEC = 1000; + private static final int MILLIS_IN_SEC = 1000; - private StormRunner() { - } + private StormRunner() { + } - public static void runTopologyLocally(StormTopology topology, String topologyName, Config conf, int runtimeInSeconds) - throws InterruptedException { - LocalCluster cluster = new LocalCluster(); - cluster.submitTopology(topologyName, conf, topology); - Thread.sleep((long) runtimeInSeconds * MILLIS_IN_SEC); - cluster.killTopology(topologyName); - cluster.shutdown(); - } + public static void runTopologyLocally(StormTopology topology, String topologyName, Config conf, int runtimeInSeconds) + throws InterruptedException { + LocalCluster cluster = new LocalCluster(); + cluster.submitTopology(topologyName, conf, topology); + Thread.sleep((long) runtimeInSeconds * MILLIS_IN_SEC); + cluster.killTopology(topologyName); + cluster.shutdown(); + } } diff --git a/src/jvm/storm/starter/util/TupleHelpers.java b/src/jvm/storm/starter/util/TupleHelpers.java index 58693d77ad..a46480e1bd 100644 --- a/src/jvm/storm/starter/util/TupleHelpers.java +++ b/src/jvm/storm/starter/util/TupleHelpers.java @@ -5,12 +5,12 @@ public final class TupleHelpers { - private TupleHelpers() { - } + private TupleHelpers() { + } - public static boolean isTickTuple(Tuple tuple) { - return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID) - && tuple.getSourceStreamId().equals(Constants.SYSTEM_TICK_STREAM_ID); - } + public static boolean isTickTuple(Tuple tuple) { + return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID) && tuple.getSourceStreamId().equals( + Constants.SYSTEM_TICK_STREAM_ID); + } } diff --git a/test/jvm/storm/starter/bolt/IntermediateRankingsBoltTest.java b/test/jvm/storm/starter/bolt/IntermediateRankingsBoltTest.java index c51d0816a7..cbbee3ce80 100644 --- a/test/jvm/storm/starter/bolt/IntermediateRankingsBoltTest.java +++ b/test/jvm/storm/starter/bolt/IntermediateRankingsBoltTest.java @@ -1,135 +1,129 @@ package storm.starter.bolt; -import static org.fest.assertions.api.Assertions.assertThat; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; -import static org.mockito.Mockito.when; - -import java.util.Map; - -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import storm.starter.tools.MockTupleHelpers; import backtype.storm.Config; import backtype.storm.topology.BasicOutputCollector; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; - import com.google.common.collect.Lists; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; +import storm.starter.tools.MockTupleHelpers; + +import java.util.Map; + +import static org.fest.assertions.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.*; public class IntermediateRankingsBoltTest { - private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id"; - private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id"; - private static final Object ANY_OBJECT = new Object(); - private static final int ANY_TOPN = 10; - private static final long ANY_COUNT = 42; - - private Tuple mockRankableTuple(Object obj, long count) { - Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID); - when(tuple.getValues()).thenReturn(Lists.newArrayList(ANY_OBJECT, ANY_COUNT)); - return tuple; - } - - @DataProvider - public Object[][] illegalTopN() { - return new Object[][] { { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; - } - - @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopN") - public void negativeOrZeroTopNShouldThrowIAE(int topN) { - new IntermediateRankingsBolt(topN); - } - - @DataProvider - public Object[][] illegalEmitFrequency() { - return new Object[][] { { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; - } - - @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalEmitFrequency") - public void negativeOrZeroEmitFrequencyShouldThrowIAE(int emitFrequencyInSeconds) { - new IntermediateRankingsBolt(ANY_TOPN, emitFrequencyInSeconds); - } - - @DataProvider - public Object[][] legalTopN() { - return new Object[][] { { 1 }, { 2 }, { 3 }, { 20 } }; - } - - @Test(dataProvider = "legalTopN") - public void positiveTopNShouldBeOk(int topN) { - new IntermediateRankingsBolt(topN); - } - - @DataProvider - public Object[][] legalEmitFrequency() { - return new Object[][] { { 1 }, { 2 }, { 3 }, { 20 } }; - } - - @Test(dataProvider = "legalEmitFrequency") - public void positiveEmitFrequencyShouldBeOk(int emitFrequencyInSeconds) { - new IntermediateRankingsBolt(ANY_TOPN, emitFrequencyInSeconds); - } - - @Test - public void shouldEmitSomethingIfTickTupleIsReceived() { - // given - Tuple tickTuple = MockTupleHelpers.mockTickTuple(); - BasicOutputCollector collector = mock(BasicOutputCollector.class); - IntermediateRankingsBolt bolt = new IntermediateRankingsBolt(); - - // when - bolt.execute(tickTuple, collector); - - // then - // verifyZeroInteractions(collector); - verify(collector).emit(any(Values.class)); - } - - @Test - public void shouldEmitNothingIfNormalTupleIsReceived() { - // given - Tuple normalTuple = mockRankableTuple(ANY_OBJECT, ANY_COUNT); - BasicOutputCollector collector = mock(BasicOutputCollector.class); - IntermediateRankingsBolt bolt = new IntermediateRankingsBolt(); - - // when - bolt.execute(normalTuple, collector); - - // then - verifyZeroInteractions(collector); - } - - @Test - public void shouldDeclareOutputFields() { - // given - OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); - IntermediateRankingsBolt bolt = new IntermediateRankingsBolt(); - - // when - bolt.declareOutputFields(declarer); - - // then - verify(declarer, times(1)).declare(any(Fields.class)); - } - - @Test - public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() { - // given - IntermediateRankingsBolt bolt = new IntermediateRankingsBolt(); - - // when - Map componentConfig = bolt.getComponentConfiguration(); - - // then - assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); - Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); - assertThat(emitFrequencyInSeconds).isGreaterThan(0); - } + private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id"; + private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id"; + private static final Object ANY_OBJECT = new Object(); + private static final int ANY_TOPN = 10; + private static final long ANY_COUNT = 42; + + private Tuple mockRankableTuple(Object obj, long count) { + Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID); + when(tuple.getValues()).thenReturn(Lists.newArrayList(ANY_OBJECT, ANY_COUNT)); + return tuple; + } + + @DataProvider + public Object[][] illegalTopN() { + return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopN") + public void negativeOrZeroTopNShouldThrowIAE(int topN) { + new IntermediateRankingsBolt(topN); + } + + @DataProvider + public Object[][] illegalEmitFrequency() { + return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalEmitFrequency") + public void negativeOrZeroEmitFrequencyShouldThrowIAE(int emitFrequencyInSeconds) { + new IntermediateRankingsBolt(ANY_TOPN, emitFrequencyInSeconds); + } + + @DataProvider + public Object[][] legalTopN() { + return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } }; + } + + @Test(dataProvider = "legalTopN") + public void positiveTopNShouldBeOk(int topN) { + new IntermediateRankingsBolt(topN); + } + + @DataProvider + public Object[][] legalEmitFrequency() { + return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } }; + } + + @Test(dataProvider = "legalEmitFrequency") + public void positiveEmitFrequencyShouldBeOk(int emitFrequencyInSeconds) { + new IntermediateRankingsBolt(ANY_TOPN, emitFrequencyInSeconds); + } + + @Test + public void shouldEmitSomethingIfTickTupleIsReceived() { + // given + Tuple tickTuple = MockTupleHelpers.mockTickTuple(); + BasicOutputCollector collector = mock(BasicOutputCollector.class); + IntermediateRankingsBolt bolt = new IntermediateRankingsBolt(); + + // when + bolt.execute(tickTuple, collector); + + // then + // verifyZeroInteractions(collector); + verify(collector).emit(any(Values.class)); + } + + @Test + public void shouldEmitNothingIfNormalTupleIsReceived() { + // given + Tuple normalTuple = mockRankableTuple(ANY_OBJECT, ANY_COUNT); + BasicOutputCollector collector = mock(BasicOutputCollector.class); + IntermediateRankingsBolt bolt = new IntermediateRankingsBolt(); + + // when + bolt.execute(normalTuple, collector); + + // then + verifyZeroInteractions(collector); + } + + @Test + public void shouldDeclareOutputFields() { + // given + OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); + IntermediateRankingsBolt bolt = new IntermediateRankingsBolt(); + + // when + bolt.declareOutputFields(declarer); + + // then + verify(declarer, times(1)).declare(any(Fields.class)); + } + + @Test + public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() { + // given + IntermediateRankingsBolt bolt = new IntermediateRankingsBolt(); + + // when + Map componentConfig = bolt.getComponentConfiguration(); + + // then + assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + assertThat(emitFrequencyInSeconds).isGreaterThan(0); + } } diff --git a/test/jvm/storm/starter/bolt/RollingCountBoltTest.java b/test/jvm/storm/starter/bolt/RollingCountBoltTest.java index ae19041e82..c9a516a385 100644 --- a/test/jvm/storm/starter/bolt/RollingCountBoltTest.java +++ b/test/jvm/storm/starter/bolt/RollingCountBoltTest.java @@ -1,18 +1,5 @@ package storm.starter.bolt; -import static org.fest.assertions.api.Assertions.assertThat; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; -import static org.mockito.Mockito.when; - -import java.util.Map; - -import org.testng.annotations.Test; - -import storm.starter.tools.MockTupleHelpers; import backtype.storm.Config; import backtype.storm.task.OutputCollector; import backtype.storm.task.TopologyContext; @@ -20,82 +7,90 @@ import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; +import org.testng.annotations.Test; +import storm.starter.tools.MockTupleHelpers; + +import java.util.Map; + +import static org.fest.assertions.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.*; public class RollingCountBoltTest { - private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id"; - private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id"; - - private Tuple mockNormalTuple(Object obj) { - Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID); - when(tuple.getValue(0)).thenReturn(obj); - return tuple; - } - - @SuppressWarnings("rawtypes") - @Test - public void shouldEmitNothingIfNoObjectHasBeenCountedYetAndTickTupleIsReceived() { - // given - Tuple tickTuple = MockTupleHelpers.mockTickTuple(); - RollingCountBolt bolt = new RollingCountBolt(); - Map conf = mock(Map.class); - TopologyContext context = mock(TopologyContext.class); - OutputCollector collector = mock(OutputCollector.class); - bolt.prepare(conf, context, collector); - - // when - bolt.execute(tickTuple); - - // then - verifyZeroInteractions(collector); - } - - @SuppressWarnings("rawtypes") - @Test - public void shouldEmitSomethingIfAtLeastOneObjectWasCountedAndTickTupleIsReceived() { - // given - Tuple normalTuple = mockNormalTuple(new Object()); - Tuple tickTuple = MockTupleHelpers.mockTickTuple(); - - RollingCountBolt bolt = new RollingCountBolt(); - Map conf = mock(Map.class); - TopologyContext context = mock(TopologyContext.class); - OutputCollector collector = mock(OutputCollector.class); - bolt.prepare(conf, context, collector); - - // when - bolt.execute(normalTuple); - bolt.execute(tickTuple); - - // then - verify(collector).emit(any(Values.class)); - } - - @Test - public void shouldDeclareOutputFields() { - // given - OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); - RollingCountBolt bolt = new RollingCountBolt(); - - // when - bolt.declareOutputFields(declarer); - - // then - verify(declarer, times(1)).declare(any(Fields.class)); - - } - - @Test - public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() { - // given - RollingCountBolt bolt = new RollingCountBolt(); - - // when - Map componentConfig = bolt.getComponentConfiguration(); - - // then - assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); - Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); - assertThat(emitFrequencyInSeconds).isGreaterThan(0); - } + private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id"; + private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id"; + + private Tuple mockNormalTuple(Object obj) { + Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID); + when(tuple.getValue(0)).thenReturn(obj); + return tuple; + } + + @SuppressWarnings("rawtypes") + @Test + public void shouldEmitNothingIfNoObjectHasBeenCountedYetAndTickTupleIsReceived() { + // given + Tuple tickTuple = MockTupleHelpers.mockTickTuple(); + RollingCountBolt bolt = new RollingCountBolt(); + Map conf = mock(Map.class); + TopologyContext context = mock(TopologyContext.class); + OutputCollector collector = mock(OutputCollector.class); + bolt.prepare(conf, context, collector); + + // when + bolt.execute(tickTuple); + + // then + verifyZeroInteractions(collector); + } + + @SuppressWarnings("rawtypes") + @Test + public void shouldEmitSomethingIfAtLeastOneObjectWasCountedAndTickTupleIsReceived() { + // given + Tuple normalTuple = mockNormalTuple(new Object()); + Tuple tickTuple = MockTupleHelpers.mockTickTuple(); + + RollingCountBolt bolt = new RollingCountBolt(); + Map conf = mock(Map.class); + TopologyContext context = mock(TopologyContext.class); + OutputCollector collector = mock(OutputCollector.class); + bolt.prepare(conf, context, collector); + + // when + bolt.execute(normalTuple); + bolt.execute(tickTuple); + + // then + verify(collector).emit(any(Values.class)); + } + + @Test + public void shouldDeclareOutputFields() { + // given + OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); + RollingCountBolt bolt = new RollingCountBolt(); + + // when + bolt.declareOutputFields(declarer); + + // then + verify(declarer, times(1)).declare(any(Fields.class)); + + } + + @Test + public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() { + // given + RollingCountBolt bolt = new RollingCountBolt(); + + // when + Map componentConfig = bolt.getComponentConfiguration(); + + // then + assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + assertThat(emitFrequencyInSeconds).isGreaterThan(0); + } } diff --git a/test/jvm/storm/starter/bolt/TotalRankingsBoltTest.java b/test/jvm/storm/starter/bolt/TotalRankingsBoltTest.java index 69f531950f..a8bed45f8f 100644 --- a/test/jvm/storm/starter/bolt/TotalRankingsBoltTest.java +++ b/test/jvm/storm/starter/bolt/TotalRankingsBoltTest.java @@ -1,135 +1,130 @@ package storm.starter.bolt; -import static org.fest.assertions.api.Assertions.assertThat; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; -import static org.mockito.Mockito.when; - -import java.util.Map; - -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import storm.starter.tools.MockTupleHelpers; -import storm.starter.tools.Rankings; import backtype.storm.Config; import backtype.storm.topology.BasicOutputCollector; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; +import storm.starter.tools.MockTupleHelpers; +import storm.starter.tools.Rankings; + +import java.util.Map; + +import static org.fest.assertions.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.*; public class TotalRankingsBoltTest { - private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id"; - private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id"; - private static final Object ANY_OBJECT = new Object(); - private static final int ANY_TOPN = 10; - private static final long ANY_COUNT = 42; - - private Tuple mockRankingsTuple(Object obj, long count) { - Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID); - Rankings rankings = mock(Rankings.class); - when(tuple.getValue(0)).thenReturn(rankings); - return tuple; - } - - @DataProvider - public Object[][] illegalTopN() { - return new Object[][] { { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; - } - - @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopN") - public void negativeOrZeroTopNShouldThrowIAE(int topN) { - new TotalRankingsBolt(topN); - } - - @DataProvider - public Object[][] illegalEmitFrequency() { - return new Object[][] { { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; - } - - @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalEmitFrequency") - public void negativeOrZeroEmitFrequencyShouldThrowIAE(int emitFrequencyInSeconds) { - new TotalRankingsBolt(ANY_TOPN, emitFrequencyInSeconds); - } - - @DataProvider - public Object[][] legalTopN() { - return new Object[][] { { 1 }, { 2 }, { 3 }, { 20 } }; - } - - @Test(dataProvider = "legalTopN") - public void positiveTopNShouldBeOk(int topN) { - new TotalRankingsBolt(topN); - } - - @DataProvider - public Object[][] legalEmitFrequency() { - return new Object[][] { { 1 }, { 2 }, { 3 }, { 20 } }; - } - - @Test(dataProvider = "legalEmitFrequency") - public void positiveEmitFrequencyShouldBeOk(int emitFrequencyInSeconds) { - new TotalRankingsBolt(ANY_TOPN, emitFrequencyInSeconds); - } - - @Test - public void shouldEmitSomethingIfTickTupleIsReceived() { - // given - Tuple tickTuple = MockTupleHelpers.mockTickTuple(); - BasicOutputCollector collector = mock(BasicOutputCollector.class); - TotalRankingsBolt bolt = new TotalRankingsBolt(); - - // when - bolt.execute(tickTuple, collector); - - // then - // verifyZeroInteractions(collector); - verify(collector).emit(any(Values.class)); - } - - @Test - public void shouldEmitNothingIfNormalTupleIsReceived() { - // given - Tuple normalTuple = mockRankingsTuple(ANY_OBJECT, ANY_COUNT); - BasicOutputCollector collector = mock(BasicOutputCollector.class); - TotalRankingsBolt bolt = new TotalRankingsBolt(); - - // when - bolt.execute(normalTuple, collector); - - // then - verifyZeroInteractions(collector); - } - - @Test - public void shouldDeclareOutputFields() { - // given - OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); - TotalRankingsBolt bolt = new TotalRankingsBolt(); - - // when - bolt.declareOutputFields(declarer); - - // then - verify(declarer, times(1)).declare(any(Fields.class)); - } - - @Test - public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() { - // given - TotalRankingsBolt bolt = new TotalRankingsBolt(); - - // when - Map componentConfig = bolt.getComponentConfiguration(); - - // then - assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); - Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); - assertThat(emitFrequencyInSeconds).isGreaterThan(0); - } + private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id"; + private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id"; + private static final Object ANY_OBJECT = new Object(); + private static final int ANY_TOPN = 10; + private static final long ANY_COUNT = 42; + + private Tuple mockRankingsTuple(Object obj, long count) { + Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID); + Rankings rankings = mock(Rankings.class); + when(tuple.getValue(0)).thenReturn(rankings); + return tuple; + } + + @DataProvider + public Object[][] illegalTopN() { + return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopN") + public void negativeOrZeroTopNShouldThrowIAE(int topN) { + new TotalRankingsBolt(topN); + } + + @DataProvider + public Object[][] illegalEmitFrequency() { + return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalEmitFrequency") + public void negativeOrZeroEmitFrequencyShouldThrowIAE(int emitFrequencyInSeconds) { + new TotalRankingsBolt(ANY_TOPN, emitFrequencyInSeconds); + } + + @DataProvider + public Object[][] legalTopN() { + return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } }; + } + + @Test(dataProvider = "legalTopN") + public void positiveTopNShouldBeOk(int topN) { + new TotalRankingsBolt(topN); + } + + @DataProvider + public Object[][] legalEmitFrequency() { + return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } }; + } + + @Test(dataProvider = "legalEmitFrequency") + public void positiveEmitFrequencyShouldBeOk(int emitFrequencyInSeconds) { + new TotalRankingsBolt(ANY_TOPN, emitFrequencyInSeconds); + } + + @Test + public void shouldEmitSomethingIfTickTupleIsReceived() { + // given + Tuple tickTuple = MockTupleHelpers.mockTickTuple(); + BasicOutputCollector collector = mock(BasicOutputCollector.class); + TotalRankingsBolt bolt = new TotalRankingsBolt(); + + // when + bolt.execute(tickTuple, collector); + + // then + // verifyZeroInteractions(collector); + verify(collector).emit(any(Values.class)); + } + + @Test + public void shouldEmitNothingIfNormalTupleIsReceived() { + // given + Tuple normalTuple = mockRankingsTuple(ANY_OBJECT, ANY_COUNT); + BasicOutputCollector collector = mock(BasicOutputCollector.class); + TotalRankingsBolt bolt = new TotalRankingsBolt(); + + // when + bolt.execute(normalTuple, collector); + + // then + verifyZeroInteractions(collector); + } + + @Test + public void shouldDeclareOutputFields() { + // given + OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); + TotalRankingsBolt bolt = new TotalRankingsBolt(); + + // when + bolt.declareOutputFields(declarer); + + // then + verify(declarer, times(1)).declare(any(Fields.class)); + } + + @Test + public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() { + // given + TotalRankingsBolt bolt = new TotalRankingsBolt(); + + // when + Map componentConfig = bolt.getComponentConfiguration(); + + // then + assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + assertThat(emitFrequencyInSeconds).isGreaterThan(0); + } } diff --git a/test/jvm/storm/starter/tools/MockTupleHelpers.java b/test/jvm/storm/starter/tools/MockTupleHelpers.java index d0a9e419e7..fd7d921407 100644 --- a/test/jvm/storm/starter/tools/MockTupleHelpers.java +++ b/test/jvm/storm/starter/tools/MockTupleHelpers.java @@ -1,23 +1,23 @@ package storm.starter.tools; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import backtype.storm.Constants; import backtype.storm.tuple.Tuple; +import static org.mockito.Mockito.*; + public final class MockTupleHelpers { - private MockTupleHelpers() { - } + private MockTupleHelpers() { + } - public static Tuple mockTickTuple() { - return mockTuple(Constants.SYSTEM_COMPONENT_ID, Constants.SYSTEM_TICK_STREAM_ID); - } + public static Tuple mockTickTuple() { + return mockTuple(Constants.SYSTEM_COMPONENT_ID, Constants.SYSTEM_TICK_STREAM_ID); + } - public static Tuple mockTuple(String componentId, String streamId) { - Tuple tuple = mock(Tuple.class); - when(tuple.getSourceComponent()).thenReturn(componentId); - when(tuple.getSourceStreamId()).thenReturn(streamId); - return tuple; - } + public static Tuple mockTuple(String componentId, String streamId) { + Tuple tuple = mock(Tuple.class); + when(tuple.getSourceComponent()).thenReturn(componentId); + when(tuple.getSourceStreamId()).thenReturn(streamId); + return tuple; + } } diff --git a/test/jvm/storm/starter/tools/NthLastModifiedTimeTrackerTest.java b/test/jvm/storm/starter/tools/NthLastModifiedTimeTrackerTest.java index 77f8f717d2..c79d3825ef 100644 --- a/test/jvm/storm/starter/tools/NthLastModifiedTimeTrackerTest.java +++ b/test/jvm/storm/starter/tools/NthLastModifiedTimeTrackerTest.java @@ -1,112 +1,108 @@ package storm.starter.tools; -import static org.fest.assertions.api.Assertions.assertThat; - +import backtype.storm.utils.Time; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import backtype.storm.utils.Time; +import static org.fest.assertions.api.Assertions.assertThat; public class NthLastModifiedTimeTrackerTest { - private static final int ANY_NUM_TIMES_TO_TRACK = 3; - private static final int MILLIS_IN_SEC = 1000; - - @DataProvider - public Object[][] illegalNumTimesData() { - return new Object[][] { { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; - } - - @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalNumTimesData") - public void negativeOrZeroNumTimesToTrackShouldThrowIAE(int numTimesToTrack) { - new NthLastModifiedTimeTracker(numTimesToTrack); - } - - @DataProvider - public Object[][] legalNumTimesData() { - return new Object[][] { { 1 }, { 2 }, { 3 }, { 20 } }; - } - - @Test(dataProvider = "legalNumTimesData") - public void positiveNumTimesToTrackShouldBeOk(int numTimesToTrack) { - new NthLastModifiedTimeTracker(numTimesToTrack); - } - - @DataProvider - public Object[][] whenNotYetMarkedAsModifiedData() { - return new Object[][] { { 0 }, { 1 }, { 2 }, { 3 }, { 4 }, { 5 }, { 8 }, { 10 } }; + private static final int ANY_NUM_TIMES_TO_TRACK = 3; + private static final int MILLIS_IN_SEC = 1000; + + @DataProvider + public Object[][] illegalNumTimesData() { + return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalNumTimesData") + public void negativeOrZeroNumTimesToTrackShouldThrowIAE(int numTimesToTrack) { + new NthLastModifiedTimeTracker(numTimesToTrack); + } + + @DataProvider + public Object[][] legalNumTimesData() { + return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } }; + } + + @Test(dataProvider = "legalNumTimesData") + public void positiveNumTimesToTrackShouldBeOk(int numTimesToTrack) { + new NthLastModifiedTimeTracker(numTimesToTrack); + } + + @DataProvider + public Object[][] whenNotYetMarkedAsModifiedData() { + return new Object[][]{ { 0 }, { 1 }, { 2 }, { 3 }, { 4 }, { 5 }, { 8 }, { 10 } }; + } + + @Test(dataProvider = "whenNotYetMarkedAsModifiedData") + public void shouldReturnCorrectModifiedTimeEvenWhenNotYetMarkedAsModified(int secondsToAdvance) { + // given + Time.startSimulating(); + NthLastModifiedTimeTracker tracker = new NthLastModifiedTimeTracker(ANY_NUM_TIMES_TO_TRACK); + + // when + advanceSimulatedTimeBy(secondsToAdvance); + int seconds = tracker.secondsSinceOldestModification(); + + // then + assertThat(seconds).isEqualTo(secondsToAdvance); + + // cleanup + Time.stopSimulating(); + } + + @DataProvider + public Object[][] simulatedTrackerIterations() { + return new Object[][]{ { 1, new int[]{ 0, 1 }, new int[]{ 0, 0 } }, { 1, new int[]{ 0, 2 }, new int[]{ 0, 0 } }, + { 2, new int[]{ 2, 2 }, new int[]{ 2, 2 } }, { 2, new int[]{ 0, 4 }, new int[]{ 0, 4 } }, + { 1, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 0, 0, 0, 0, 0, 0, 0 } }, + { 1, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 0, 0, 0, 0, 0, 0, 0 } }, + { 2, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 1, 1, 1, 1, 1, 1 } }, + { 2, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 2, 2, 2, 2, 2, 2 } }, + { 2, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 2, 3, 4, 5, 6, 7 } }, + { 3, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 2, 2, 2, 2, 2, 2 } }, + { 3, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 3, 5, 7, 9, 11, 13 } }, + { 3, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 4, 4, 4, 4, 4, 4 } }, + { 4, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 2, 3, 3, 3, 3, 3 } }, + { 4, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 3, 6, 9, 12, 15, 18 } }, + { 4, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 4, 6, 6, 6, 6, 6 } }, + { 5, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 2, 3, 4, 4, 4, 4 } }, + { 5, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 3, 6, 10, 14, 18, 22 } }, + { 5, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 4, 6, 8, 8, 8, 8 } }, + { 6, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 2, 3, 4, 5, 5, 5 } }, + { 6, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 3, 6, 10, 15, 20, 25 } }, + { 6, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 4, 6, 8, 10, 10, 10 } }, + { 3, new int[]{ 1, 2, 3 }, new int[]{ 1, 3, 5 } } }; + } + + @Test(dataProvider = "simulatedTrackerIterations") + public void shouldReturnCorrectModifiedTimeWhenMarkedAsModified(int numTimesToTrack, + int[] secondsToAdvancePerIteration, int[] expLastModifiedTimes) { + // given + Time.startSimulating(); + NthLastModifiedTimeTracker tracker = new NthLastModifiedTimeTracker(numTimesToTrack); + + int[] modifiedTimes = new int[expLastModifiedTimes.length]; + + // when + int i = 0; + for (int secondsToAdvance : secondsToAdvancePerIteration) { + advanceSimulatedTimeBy(secondsToAdvance); + tracker.markAsModified(); + modifiedTimes[i] = tracker.secondsSinceOldestModification(); + i++; } - @Test(dataProvider = "whenNotYetMarkedAsModifiedData") - public void shouldReturnCorrectModifiedTimeEvenWhenNotYetMarkedAsModified(int secondsToAdvance) { - // given - Time.startSimulating(); - NthLastModifiedTimeTracker tracker = new NthLastModifiedTimeTracker(ANY_NUM_TIMES_TO_TRACK); + // then + assertThat(modifiedTimes).isEqualTo(expLastModifiedTimes); - // when - advanceSimulatedTimeBy(secondsToAdvance); - int seconds = tracker.secondsSinceOldestModification(); - - // then - assertThat(seconds).isEqualTo(secondsToAdvance); - - // cleanup - Time.stopSimulating(); - } + // cleanup + Time.stopSimulating(); + } - @DataProvider - public Object[][] simulatedTrackerIterations() { - return new Object[][] { - { 1, new int[] { 0, 1 }, new int[] { 0, 0 } }, - { 1, new int[] { 0, 2 }, new int[] { 0, 0 } }, - { 2, new int[] { 2, 2 }, new int[] { 2, 2 } }, - { 2, new int[] { 0, 4 }, new int[] { 0, 4 } }, - { 1, new int[] { 1, 1, 1, 1, 1, 1, 1 }, new int[] { 0, 0, 0, 0, 0, 0, 0 } }, - { 1, new int[] { 1, 2, 3, 4, 5, 6, 7 }, new int[] { 0, 0, 0, 0, 0, 0, 0 } }, - { 2, new int[] { 1, 1, 1, 1, 1, 1, 1 }, new int[] { 1, 1, 1, 1, 1, 1, 1 } }, - { 2, new int[] { 2, 2, 2, 2, 2, 2, 2 }, new int[] { 2, 2, 2, 2, 2, 2, 2 } }, - { 2, new int[] { 1, 2, 3, 4, 5, 6, 7 }, new int[] { 1, 2, 3, 4, 5, 6, 7 } }, - { 3, new int[] { 1, 1, 1, 1, 1, 1, 1 }, new int[] { 1, 2, 2, 2, 2, 2, 2 } }, - { 3, new int[] { 1, 2, 3, 4, 5, 6, 7 }, new int[] { 1, 3, 5, 7, 9, 11, 13 } }, - { 3, new int[] { 2, 2, 2, 2, 2, 2, 2 }, new int[] { 2, 4, 4, 4, 4, 4, 4 } }, - { 4, new int[] { 1, 1, 1, 1, 1, 1, 1 }, new int[] { 1, 2, 3, 3, 3, 3, 3 } }, - { 4, new int[] { 1, 2, 3, 4, 5, 6, 7 }, new int[] { 1, 3, 6, 9, 12, 15, 18 } }, - { 4, new int[] { 2, 2, 2, 2, 2, 2, 2 }, new int[] { 2, 4, 6, 6, 6, 6, 6 } }, - { 5, new int[] { 1, 1, 1, 1, 1, 1, 1 }, new int[] { 1, 2, 3, 4, 4, 4, 4 } }, - { 5, new int[] { 1, 2, 3, 4, 5, 6, 7 }, new int[] { 1, 3, 6, 10, 14, 18, 22 } }, - { 5, new int[] { 2, 2, 2, 2, 2, 2, 2 }, new int[] { 2, 4, 6, 8, 8, 8, 8 } }, - { 6, new int[] { 1, 1, 1, 1, 1, 1, 1 }, new int[] { 1, 2, 3, 4, 5, 5, 5 } }, - { 6, new int[] { 1, 2, 3, 4, 5, 6, 7 }, new int[] { 1, 3, 6, 10, 15, 20, 25 } }, - { 6, new int[] { 2, 2, 2, 2, 2, 2, 2 }, new int[] { 2, 4, 6, 8, 10, 10, 10 } }, - { 3, new int[] { 1, 2, 3 }, new int[] { 1, 3, 5 } } }; - } - - @Test(dataProvider = "simulatedTrackerIterations") - public void shouldReturnCorrectModifiedTimeWhenMarkedAsModified(int numTimesToTrack, - int[] secondsToAdvancePerIteration, int[] expLastModifiedTimes) { - // given - Time.startSimulating(); - NthLastModifiedTimeTracker tracker = new NthLastModifiedTimeTracker(numTimesToTrack); - - int[] modifiedTimes = new int[expLastModifiedTimes.length]; - - // when - int i = 0; - for (int secondsToAdvance : secondsToAdvancePerIteration) { - advanceSimulatedTimeBy(secondsToAdvance); - tracker.markAsModified(); - modifiedTimes[i] = tracker.secondsSinceOldestModification(); - i++; - } - - // then - assertThat(modifiedTimes).isEqualTo(expLastModifiedTimes); - - // cleanup - Time.stopSimulating(); - } - - private void advanceSimulatedTimeBy(int seconds) { - Time.advanceTime(seconds * MILLIS_IN_SEC); - } + private void advanceSimulatedTimeBy(int seconds) { + Time.advanceTime(seconds * MILLIS_IN_SEC); + } } diff --git a/test/jvm/storm/starter/tools/RankableObjectWithFieldsTest.java b/test/jvm/storm/starter/tools/RankableObjectWithFieldsTest.java index b2360c13f1..c8a14ccc03 100644 --- a/test/jvm/storm/starter/tools/RankableObjectWithFieldsTest.java +++ b/test/jvm/storm/starter/tools/RankableObjectWithFieldsTest.java @@ -1,225 +1,214 @@ package storm.starter.tools; -import static org.fest.assertions.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.List; - +import backtype.storm.tuple.Tuple; +import com.google.common.collect.Lists; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import backtype.storm.tuple.Tuple; +import java.util.ArrayList; +import java.util.List; -import com.google.common.collect.Lists; +import static org.fest.assertions.api.Assertions.assertThat; +import static org.mockito.Mockito.*; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; public class RankableObjectWithFieldsTest { - private static final Object ANY_OBJECT = new Object(); - private static final long ANY_COUNT = 271; - private static final String ANY_FIELD = "someAdditionalField"; - private static final int GREATER_THAN = 1; - private static final int EQUAL_TO = 0; - private static final int SMALLER_THAN = -1; - - @Test(expectedExceptions = IllegalArgumentException.class) - public void constructorWithNullObjectAndNoFieldsShouldThrowIAE() { - new RankableObjectWithFields(null, ANY_COUNT); - } - - @Test(expectedExceptions = IllegalArgumentException.class) - public void constructorWithNullObjectAndFieldsShouldThrowIAE() { - Object someAdditionalField = new Object(); - new RankableObjectWithFields(null, ANY_COUNT, someAdditionalField); - } - - @Test(expectedExceptions = IllegalArgumentException.class) - public void constructorWithNegativeCountAndNoFieldsShouldThrowIAE() { - new RankableObjectWithFields(ANY_OBJECT, -1); - } - - @Test(expectedExceptions = IllegalArgumentException.class) - public void constructorWithNegativeCountAndFieldsShouldThrowIAE() { - Object someAdditionalField = new Object(); - new RankableObjectWithFields(ANY_OBJECT, -1, someAdditionalField); - } - - @Test - public void shouldBeEqualToItself() { - RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT); - assertThat(r).isEqualTo(r); - } - - @DataProvider - public Object[][] otherClassesData() { - return new Object[][] { - { new String("foo") }, - { new Object() }, - { Integer.valueOf(4) }, - { Lists.newArrayList(7, 8, 9) } }; - } - - @Test(dataProvider = "otherClassesData") - public void shouldNotBeEqualToInstancesOfOtherClasses(Object notARankable) { - RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT); - assertFalse(r.equals(notARankable), r + " is equal to " + notARankable + " but it should not be"); - } - - @DataProvider - public Object[][] falseDuplicatesData() { - return new Object[][] { - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1) }, - { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("Foo", 1) }, - { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("FOO", 1) }, - { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 1) }, - { new RankableObjectWithFields("", 0), new RankableObjectWithFields("", 1) }, - { new RankableObjectWithFields("", 1), new RankableObjectWithFields("bar", 1) } }; - } - - @Test(dataProvider = "falseDuplicatesData") - public void shouldNotBeEqualToFalseDuplicates(RankableObjectWithFields r, RankableObjectWithFields falseDuplicate) { - assertFalse(r.equals(falseDuplicate), r + " is equal to " + falseDuplicate + " but it should not be"); - } - - @Test(dataProvider = "falseDuplicatesData") - public void shouldHaveDifferentHashCodeThanFalseDuplicates(RankableObjectWithFields r, - RankableObjectWithFields falseDuplicate) { - assertThat(r.hashCode()).isNotEqualTo(falseDuplicate.hashCode()); - } - - @DataProvider - public Object[][] trueDuplicatesData() { - return new Object[][] { - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0) }, - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0, "someOtherField") }, - { - new RankableObjectWithFields("foo", 0, "someField"), - new RankableObjectWithFields("foo", 0, "someOtherField") } }; - } - - @Test(dataProvider = "trueDuplicatesData") - public void shouldBeEqualToTrueDuplicates(RankableObjectWithFields r, RankableObjectWithFields trueDuplicate) { - assertTrue(r.equals(trueDuplicate), r + " is not equal to " + trueDuplicate + " but it should be"); - } - - @Test(dataProvider = "trueDuplicatesData") - public void shouldHaveSameHashCodeAsTrueDuplicates(RankableObjectWithFields r, - RankableObjectWithFields trueDuplicate) { - assertThat(r.hashCode()).isEqualTo(trueDuplicate.hashCode()); - } - - @DataProvider - public Object[][] compareToData() { - return new Object[][] { - { new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("foo", 0), GREATER_THAN }, - { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("foo", 0), GREATER_THAN }, - { new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("bar", 0), GREATER_THAN }, - { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 0), GREATER_THAN }, - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0), EQUAL_TO }, - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 0), EQUAL_TO }, - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1000), SMALLER_THAN }, - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1), SMALLER_THAN }, - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 1), SMALLER_THAN }, - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 1000), SMALLER_THAN }, }; - } - - @Test(dataProvider = "compareToData") - public void verifyCompareTo(RankableObjectWithFields first, RankableObjectWithFields second, int expCompareToValue) { - assertThat(first.compareTo(second)).isEqualTo(expCompareToValue); - } - - @DataProvider - public Object[][] toStringData() { - return new Object[][] { { new String("foo"), 0L }, { new String("BAR"), 8L } }; - } - - @Test(dataProvider = "toStringData") - public void toStringShouldContainStringRepresentationsOfObjectAndCount(Object obj, long count) { - // given - RankableObjectWithFields r = new RankableObjectWithFields(obj, count); - - // when - String strRepresentation = r.toString(); - - // then - assertThat(strRepresentation).contains(obj.toString()).contains("" + count); - } - - @Test - public void shouldReturnTheObject() { - // given - RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD); - - // when - Object obj = r.getObject(); - - // then - assertThat(obj).isEqualTo(ANY_OBJECT); - } - - @Test - public void shouldReturnTheCount() { - // given - RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD); - - // when - long count = r.getCount(); - - // then - assertThat(count).isEqualTo(ANY_COUNT); - } - - @DataProvider - public Object[][] fieldsData() { - return new Object[][] { - { ANY_OBJECT, ANY_COUNT, new Object[] { ANY_FIELD } }, - { "quux", 42L, new Object[] { "one", "two", "three" } } }; - } - - @Test(dataProvider = "fieldsData") - public void shouldReturnTheFields(Object obj, long count, Object[] fields) { - // given - RankableObjectWithFields r = new RankableObjectWithFields(obj, count, fields); - - // when - List actualFields = r.getFields(); - - // then - assertThat(actualFields).isEqualTo(Lists.newArrayList(fields)); - } - - @Test(expectedExceptions = UnsupportedOperationException.class) - public void fieldsShouldBeImmutable() { - // given - RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD); - - // when - List fields = r.getFields(); - // try to modify the list, which should fail - fields.remove(0); - - // then (exception) - } - - @Test - public void shouldCreateRankableObjectFromTuple() { - // given - Tuple tuple = mock(Tuple.class); - List tupleValues = Lists.newArrayList(ANY_OBJECT, ANY_COUNT, ANY_FIELD); - when(tuple.getValues()).thenReturn(tupleValues); - - // when - RankableObjectWithFields r = RankableObjectWithFields.from(tuple); - - // then - assertThat(r.getObject()).isEqualTo(ANY_OBJECT); - assertThat(r.getCount()).isEqualTo(ANY_COUNT); - List fields = new ArrayList(); - fields.add(ANY_FIELD); - assertThat(r.getFields()).isEqualTo(fields); - - } + private static final Object ANY_OBJECT = new Object(); + private static final long ANY_COUNT = 271; + private static final String ANY_FIELD = "someAdditionalField"; + private static final int GREATER_THAN = 1; + private static final int EQUAL_TO = 0; + private static final int SMALLER_THAN = -1; + + @Test(expectedExceptions = IllegalArgumentException.class) + public void constructorWithNullObjectAndNoFieldsShouldThrowIAE() { + new RankableObjectWithFields(null, ANY_COUNT); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void constructorWithNullObjectAndFieldsShouldThrowIAE() { + Object someAdditionalField = new Object(); + new RankableObjectWithFields(null, ANY_COUNT, someAdditionalField); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void constructorWithNegativeCountAndNoFieldsShouldThrowIAE() { + new RankableObjectWithFields(ANY_OBJECT, -1); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void constructorWithNegativeCountAndFieldsShouldThrowIAE() { + Object someAdditionalField = new Object(); + new RankableObjectWithFields(ANY_OBJECT, -1, someAdditionalField); + } + + @Test + public void shouldBeEqualToItself() { + RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT); + assertThat(r).isEqualTo(r); + } + + @DataProvider + public Object[][] otherClassesData() { + return new Object[][]{ { new String("foo") }, { new Object() }, { Integer.valueOf(4) }, { Lists.newArrayList(7, 8, + 9) } }; + } + + @Test(dataProvider = "otherClassesData") + public void shouldNotBeEqualToInstancesOfOtherClasses(Object notARankable) { + RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT); + assertFalse(r.equals(notARankable), r + " is equal to " + notARankable + " but it should not be"); + } + + @DataProvider + public Object[][] falseDuplicatesData() { + return new Object[][]{ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1) }, + { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("Foo", 1) }, + { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("FOO", 1) }, + { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 1) }, + { new RankableObjectWithFields("", 0), new RankableObjectWithFields("", 1) }, { new RankableObjectWithFields("", + 1), new RankableObjectWithFields("bar", 1) } }; + } + + @Test(dataProvider = "falseDuplicatesData") + public void shouldNotBeEqualToFalseDuplicates(RankableObjectWithFields r, RankableObjectWithFields falseDuplicate) { + assertFalse(r.equals(falseDuplicate), r + " is equal to " + falseDuplicate + " but it should not be"); + } + + @Test(dataProvider = "falseDuplicatesData") + public void shouldHaveDifferentHashCodeThanFalseDuplicates(RankableObjectWithFields r, + RankableObjectWithFields falseDuplicate) { + assertThat(r.hashCode()).isNotEqualTo(falseDuplicate.hashCode()); + } + + @DataProvider + public Object[][] trueDuplicatesData() { + return new Object[][]{ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0) }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0, "someOtherField") }, + { new RankableObjectWithFields("foo", 0, "someField"), new RankableObjectWithFields("foo", 0, + "someOtherField") } }; + } + + @Test(dataProvider = "trueDuplicatesData") + public void shouldBeEqualToTrueDuplicates(RankableObjectWithFields r, RankableObjectWithFields trueDuplicate) { + assertTrue(r.equals(trueDuplicate), r + " is not equal to " + trueDuplicate + " but it should be"); + } + + @Test(dataProvider = "trueDuplicatesData") + public void shouldHaveSameHashCodeAsTrueDuplicates(RankableObjectWithFields r, + RankableObjectWithFields trueDuplicate) { + assertThat(r.hashCode()).isEqualTo(trueDuplicate.hashCode()); + } + + @DataProvider + public Object[][] compareToData() { + return new Object[][]{ { new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("foo", 0), + GREATER_THAN }, { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("foo", 0), + GREATER_THAN }, { new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("bar", 0), + GREATER_THAN }, { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 0), + GREATER_THAN }, { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0), EQUAL_TO }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 0), EQUAL_TO }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1000), SMALLER_THAN }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1), SMALLER_THAN }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 1), SMALLER_THAN }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 1000), SMALLER_THAN }, }; + } + + @Test(dataProvider = "compareToData") + public void verifyCompareTo(RankableObjectWithFields first, RankableObjectWithFields second, int expCompareToValue) { + assertThat(first.compareTo(second)).isEqualTo(expCompareToValue); + } + + @DataProvider + public Object[][] toStringData() { + return new Object[][]{ { new String("foo"), 0L }, { new String("BAR"), 8L } }; + } + + @Test(dataProvider = "toStringData") + public void toStringShouldContainStringRepresentationsOfObjectAndCount(Object obj, long count) { + // given + RankableObjectWithFields r = new RankableObjectWithFields(obj, count); + + // when + String strRepresentation = r.toString(); + + // then + assertThat(strRepresentation).contains(obj.toString()).contains("" + count); + } + + @Test + public void shouldReturnTheObject() { + // given + RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD); + + // when + Object obj = r.getObject(); + + // then + assertThat(obj).isEqualTo(ANY_OBJECT); + } + + @Test + public void shouldReturnTheCount() { + // given + RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD); + + // when + long count = r.getCount(); + + // then + assertThat(count).isEqualTo(ANY_COUNT); + } + + @DataProvider + public Object[][] fieldsData() { + return new Object[][]{ { ANY_OBJECT, ANY_COUNT, new Object[]{ ANY_FIELD } }, + { "quux", 42L, new Object[]{ "one", "two", "three" } } }; + } + + @Test(dataProvider = "fieldsData") + public void shouldReturnTheFields(Object obj, long count, Object[] fields) { + // given + RankableObjectWithFields r = new RankableObjectWithFields(obj, count, fields); + + // when + List actualFields = r.getFields(); + + // then + assertThat(actualFields).isEqualTo(Lists.newArrayList(fields)); + } + + @Test(expectedExceptions = UnsupportedOperationException.class) + public void fieldsShouldBeImmutable() { + // given + RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD); + + // when + List fields = r.getFields(); + // try to modify the list, which should fail + fields.remove(0); + + // then (exception) + } + + @Test + public void shouldCreateRankableObjectFromTuple() { + // given + Tuple tuple = mock(Tuple.class); + List tupleValues = Lists.newArrayList(ANY_OBJECT, ANY_COUNT, ANY_FIELD); + when(tuple.getValues()).thenReturn(tupleValues); + + // when + RankableObjectWithFields r = RankableObjectWithFields.from(tuple); + + // then + assertThat(r.getObject()).isEqualTo(ANY_OBJECT); + assertThat(r.getCount()).isEqualTo(ANY_COUNT); + List fields = new ArrayList(); + fields.add(ANY_FIELD); + assertThat(r.getFields()).isEqualTo(fields); + + } } diff --git a/test/jvm/storm/starter/tools/RankingsTest.java b/test/jvm/storm/starter/tools/RankingsTest.java index b95a86f85a..138a8b33be 100644 --- a/test/jvm/storm/starter/tools/RankingsTest.java +++ b/test/jvm/storm/starter/tools/RankingsTest.java @@ -1,248 +1,235 @@ package storm.starter.tools; -import static org.fest.assertions.api.Assertions.assertThat; - -import java.util.List; - import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import org.jmock.lib.concurrent.Blitzer; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import com.google.common.collect.Lists; - -public class RankingsTest { - - private static final int ANY_TOPN = 42; - private static final Rankable ANY_RANKABLE = new RankableObjectWithFields("someObject", ANY_TOPN); - private static final Rankable A = new RankableObjectWithFields("A", 1); - private static final Rankable B = new RankableObjectWithFields("B", 2); - private static final Rankable C = new RankableObjectWithFields("C", 3); - private static final Rankable D = new RankableObjectWithFields("D", 4); - private static final Rankable E = new RankableObjectWithFields("E", 5); - private static final Rankable F = new RankableObjectWithFields("F", 6); - private static final Rankable G = new RankableObjectWithFields("G", 7); - private static final Rankable H = new RankableObjectWithFields("H", 8); - - @DataProvider - public Object[][] illegalTopNData() { - return new Object[][] { { 0 }, { -1 }, { -2 }, { -10 } }; - } - - @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopNData") - public void constructorWithNegativeOrZeroTopNShouldThrowIAE(int topN) { - new Rankings(topN); - } - - @DataProvider - public Object[][] legalTopNData() { - return new Object[][] { { 1 }, { 2 }, { 1000 }, { 1000000 } }; - } - - @Test(dataProvider = "legalTopNData") - public void constructorWithPositiveTopNShouldBeOk(int topN) { - // given/when - Rankings rankings = new Rankings(topN); - - // then - assertThat(rankings.maxSize()).isEqualTo(topN); - } - - @Test - public void shouldHaveDefaultConstructor() { - new Rankings(); - } - - @Test - public void defaultConstructorShouldSetPositiveTopN() { - // given/when - Rankings rankings = new Rankings(); - - // then - assertThat(rankings.maxSize()).isGreaterThan(0); - } - - @DataProvider - public Object[][] rankingsGrowData() { - return new Object[][] { - { - 2, - Lists.newArrayList(new RankableObjectWithFields("A", 1), new RankableObjectWithFields("B", 2), - new RankableObjectWithFields("C", 3)) }, - { - 2, - Lists.newArrayList(new RankableObjectWithFields("A", 1), new RankableObjectWithFields("B", 2), - new RankableObjectWithFields("C", 3), new RankableObjectWithFields("D", 4)) } }; - } - - @Test(dataProvider = "rankingsGrowData") - public void sizeOfRankingsShouldNotGrowBeyondTopN(int topN, List rankables) { - // sanity check of the provided test data - assertThat(rankables.size()).overridingErrorMessage( - "The supplied test data is not correct: the number of rankables <%d> should be greater than <%d>", - rankables.size(), topN).isGreaterThan(topN); - - // given - Rankings rankings = new Rankings(topN); - - // when - for (Rankable r : rankables) { - rankings.updateWith(r); - } - - // then - assertThat(rankings.size()).isLessThanOrEqualTo(rankings.maxSize()); - } - - @DataProvider - public Object[][] simulatedRankingsData() { - return new Object[][] { - { Lists.newArrayList(A), Lists.newArrayList(A) }, - { Lists.newArrayList(B, D, A, C), Lists.newArrayList(D, C, B, A) }, - { Lists.newArrayList(B, F, A, C, D, E), Lists.newArrayList(F, E, D, C, B, A) }, - { Lists.newArrayList(G, B, F, A, C, D, E, H), Lists.newArrayList(H, G, F, E, D, C, B, A) } }; - } - - @Test(dataProvider = "simulatedRankingsData") - public void shouldCorrectlyRankWhenUpdatedWithRankables(List unsorted, List expSorted) { - // given - Rankings rankings = new Rankings(unsorted.size()); - - // when - for (Rankable r : unsorted) { - rankings.updateWith(r); - } - - // then - assertThat(rankings.getRankings()).isEqualTo(expSorted); - } - - @Test(dataProvider = "simulatedRankingsData") - public void shouldCorrectlyRankWhenEmptyAndUpdatedWithOtherRankings(List unsorted, - List expSorted) { - // given - Rankings rankings = new Rankings(unsorted.size()); - Rankings otherRankings = new Rankings(rankings.maxSize()); - for (Rankable r : unsorted) { - otherRankings.updateWith(r); - } - - // when - rankings.updateWith(otherRankings); - - // then - assertThat(rankings.getRankings()).isEqualTo(expSorted); - } - - @Test(dataProvider = "simulatedRankingsData") - public void shouldCorrectlyRankWhenUpdatedWithEmptyOtherRankings(List unsorted, List expSorted) { - // given - Rankings rankings = new Rankings(unsorted.size()); - for (Rankable r : unsorted) { - rankings.updateWith(r); - } - Rankings emptyRankings = new Rankings(ANY_TOPN); - - // when - rankings.updateWith(emptyRankings); - - // then - assertThat(rankings.getRankings()).isEqualTo(expSorted); - } - - @DataProvider - public Object[][] simulatedRankingsAndOtherRankingsData() { - return new Object[][] { - { Lists.newArrayList(A), Lists.newArrayList(A), Lists.newArrayList(A) }, - { Lists.newArrayList(A, C), Lists.newArrayList(B, D), Lists.newArrayList(D, C, B, A) }, - { Lists.newArrayList(B, F, A), Lists.newArrayList(C, D, E), Lists.newArrayList(F, E, D, C, B, A) }, - { - Lists.newArrayList(G, B, F, A, C), - Lists.newArrayList(D, E, H), - Lists.newArrayList(H, G, F, E, D, C, B, A) } }; - } - - @Test(dataProvider = "simulatedRankingsAndOtherRankingsData") - public void shouldCorrectlyRankWhenNotEmptyAndUpdatedWithOtherRankings(List unsorted, - List unsortedForOtherRankings, List expSorted) { - // given - Rankings rankings = new Rankings(expSorted.size()); - for (Rankable r : unsorted) { - rankings.updateWith(r); - } - Rankings otherRankings = new Rankings(unsortedForOtherRankings.size()); - for (Rankable r : unsortedForOtherRankings) { - otherRankings.updateWith(r); - } - - // when - rankings.updateWith(otherRankings); - - // then - assertThat(rankings.getRankings()).isEqualTo(expSorted); - } +import java.util.List; - @DataProvider - public Object[][] duplicatesData() { - Rankable A1 = new RankableObjectWithFields("A", 1); - Rankable A2 = new RankableObjectWithFields("A", 2); - Rankable A3 = new RankableObjectWithFields("A", 3); - return new Object[][] { - { Lists.newArrayList(ANY_RANKABLE, ANY_RANKABLE, ANY_RANKABLE) }, - { Lists.newArrayList(A1, A2, A3) }, }; - } +import static org.fest.assertions.api.Assertions.assertThat; - @Test(dataProvider = "duplicatesData") - public void shouldNotRankDuplicateObjectsMoreThanOnce(List duplicates) { - // given - Rankings rankings = new Rankings(duplicates.size()); +public class RankingsTest { - // when - for (Rankable r : duplicates) { + private static final int ANY_TOPN = 42; + private static final Rankable ANY_RANKABLE = new RankableObjectWithFields("someObject", ANY_TOPN); + private static final Rankable A = new RankableObjectWithFields("A", 1); + private static final Rankable B = new RankableObjectWithFields("B", 2); + private static final Rankable C = new RankableObjectWithFields("C", 3); + private static final Rankable D = new RankableObjectWithFields("D", 4); + private static final Rankable E = new RankableObjectWithFields("E", 5); + private static final Rankable F = new RankableObjectWithFields("F", 6); + private static final Rankable G = new RankableObjectWithFields("G", 7); + private static final Rankable H = new RankableObjectWithFields("H", 8); + + @DataProvider + public Object[][] illegalTopNData() { + return new Object[][]{ { 0 }, { -1 }, { -2 }, { -10 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopNData") + public void constructorWithNegativeOrZeroTopNShouldThrowIAE(int topN) { + new Rankings(topN); + } + + @DataProvider + public Object[][] legalTopNData() { + return new Object[][]{ { 1 }, { 2 }, { 1000 }, { 1000000 } }; + } + + @Test(dataProvider = "legalTopNData") + public void constructorWithPositiveTopNShouldBeOk(int topN) { + // given/when + Rankings rankings = new Rankings(topN); + + // then + assertThat(rankings.maxSize()).isEqualTo(topN); + } + + @Test + public void shouldHaveDefaultConstructor() { + new Rankings(); + } + + @Test + public void defaultConstructorShouldSetPositiveTopN() { + // given/when + Rankings rankings = new Rankings(); + + // then + assertThat(rankings.maxSize()).isGreaterThan(0); + } + + @DataProvider + public Object[][] rankingsGrowData() { + return new Object[][]{ { 2, Lists.newArrayList(new RankableObjectWithFields("A", 1), new RankableObjectWithFields( + "B", 2), new RankableObjectWithFields("C", 3)) }, { 2, Lists.newArrayList(new RankableObjectWithFields("A", 1), + new RankableObjectWithFields("B", 2), new RankableObjectWithFields("C", 3), new RankableObjectWithFields("D", + 4)) } }; + } + + @Test(dataProvider = "rankingsGrowData") + public void sizeOfRankingsShouldNotGrowBeyondTopN(int topN, List rankables) { + // sanity check of the provided test data + assertThat(rankables.size()).overridingErrorMessage( + "The supplied test data is not correct: the number of rankables <%d> should be greater than <%d>", + rankables.size(), topN).isGreaterThan(topN); + + // given + Rankings rankings = new Rankings(topN); + + // when + for (Rankable r : rankables) { + rankings.updateWith(r); + } + + // then + assertThat(rankings.size()).isLessThanOrEqualTo(rankings.maxSize()); + } + + @DataProvider + public Object[][] simulatedRankingsData() { + return new Object[][]{ { Lists.newArrayList(A), Lists.newArrayList(A) }, { Lists.newArrayList(B, D, A, C), + Lists.newArrayList(D, C, B, A) }, { Lists.newArrayList(B, F, A, C, D, E), Lists.newArrayList(F, E, D, C, B, + A) }, { Lists.newArrayList(G, B, F, A, C, D, E, H), Lists.newArrayList(H, G, F, E, D, C, B, A) } }; + } + + @Test(dataProvider = "simulatedRankingsData") + public void shouldCorrectlyRankWhenUpdatedWithRankables(List unsorted, List expSorted) { + // given + Rankings rankings = new Rankings(unsorted.size()); + + // when + for (Rankable r : unsorted) { + rankings.updateWith(r); + } + + // then + assertThat(rankings.getRankings()).isEqualTo(expSorted); + } + + @Test(dataProvider = "simulatedRankingsData") + public void shouldCorrectlyRankWhenEmptyAndUpdatedWithOtherRankings(List unsorted, + List expSorted) { + // given + Rankings rankings = new Rankings(unsorted.size()); + Rankings otherRankings = new Rankings(rankings.maxSize()); + for (Rankable r : unsorted) { + otherRankings.updateWith(r); + } + + // when + rankings.updateWith(otherRankings); + + // then + assertThat(rankings.getRankings()).isEqualTo(expSorted); + } + + @Test(dataProvider = "simulatedRankingsData") + public void shouldCorrectlyRankWhenUpdatedWithEmptyOtherRankings(List unsorted, List expSorted) { + // given + Rankings rankings = new Rankings(unsorted.size()); + for (Rankable r : unsorted) { + rankings.updateWith(r); + } + Rankings emptyRankings = new Rankings(ANY_TOPN); + + // when + rankings.updateWith(emptyRankings); + + // then + assertThat(rankings.getRankings()).isEqualTo(expSorted); + } + + @DataProvider + public Object[][] simulatedRankingsAndOtherRankingsData() { + return new Object[][]{ { Lists.newArrayList(A), Lists.newArrayList(A), Lists.newArrayList(A) }, + { Lists.newArrayList(A, C), Lists.newArrayList(B, D), Lists.newArrayList(D, C, B, A) }, { Lists.newArrayList(B, + F, A), Lists.newArrayList(C, D, E), Lists.newArrayList(F, E, D, C, B, A) }, { Lists.newArrayList(G, B, F, A, C), + Lists.newArrayList(D, E, H), Lists.newArrayList(H, G, F, E, D, C, B, A) } }; + } + + @Test(dataProvider = "simulatedRankingsAndOtherRankingsData") + public void shouldCorrectlyRankWhenNotEmptyAndUpdatedWithOtherRankings(List unsorted, + List unsortedForOtherRankings, List expSorted) { + // given + Rankings rankings = new Rankings(expSorted.size()); + for (Rankable r : unsorted) { + rankings.updateWith(r); + } + Rankings otherRankings = new Rankings(unsortedForOtherRankings.size()); + for (Rankable r : unsortedForOtherRankings) { + otherRankings.updateWith(r); + } + + // when + rankings.updateWith(otherRankings); + + // then + assertThat(rankings.getRankings()).isEqualTo(expSorted); + } + + @DataProvider + public Object[][] duplicatesData() { + Rankable A1 = new RankableObjectWithFields("A", 1); + Rankable A2 = new RankableObjectWithFields("A", 2); + Rankable A3 = new RankableObjectWithFields("A", 3); + return new Object[][]{ { Lists.newArrayList(ANY_RANKABLE, ANY_RANKABLE, ANY_RANKABLE) }, { Lists.newArrayList(A1, + A2, A3) }, }; + } + + @Test(dataProvider = "duplicatesData") + public void shouldNotRankDuplicateObjectsMoreThanOnce(List duplicates) { + // given + Rankings rankings = new Rankings(duplicates.size()); + + // when + for (Rankable r : duplicates) { + rankings.updateWith(r); + } + + // then + assertThat(rankings.size()).isEqualTo(1); + } + + @Test + public void updatingWithNewRankablesShouldBeThreadSafe() throws InterruptedException { + // given + final List entries = ImmutableList.of(A, B, C, D); + final Rankings rankings = new Rankings(entries.size()); + + // We are capturing exceptions thrown in Blitzer's child threads into this data structure so that we can properly + // pass/fail this test. The reason is that Blitzer doesn't report exceptions, which is a known bug in Blitzer + // (JMOCK-263). See https://github.com/jmock-developers/jmock-library/issues/22 for more information. + final List exceptions = Lists.newArrayList(); + Blitzer blitzer = new Blitzer(1000); + + // when + blitzer.blitz(new Runnable() { + public void run() { + for (Rankable r : entries) { + try { rankings.updateWith(r); - } - - // then - assertThat(rankings.size()).isEqualTo(1); - } - - @Test - public void updatingWithNewRankablesShouldBeThreadSafe() throws InterruptedException { - // given - final List entries = ImmutableList.of(A, B, C, D); - final Rankings rankings = new Rankings(entries.size()); - - // We are capturing exceptions thrown in Blitzer's child threads into this data structure so that we can properly - // pass/fail this test. The reason is that Blitzer doesn't report exceptions, which is a known bug in Blitzer - // (JMOCK-263). See https://github.com/jmock-developers/jmock-library/issues/22 for more information. - final List exceptions = Lists.newArrayList(); - Blitzer blitzer = new Blitzer(1000); - - // when - blitzer.blitz(new Runnable() { - public void run() { - for (Rankable r : entries) { - try { - rankings.updateWith(r); - } - catch (RuntimeException e) { - synchronized(exceptions) { - exceptions.add(e); - } - } - } - } - }); - blitzer.shutdown(); - - // then - // - if (!exceptions.isEmpty()) { - for (Exception e : exceptions) { - System.err.println(Throwables.getStackTraceAsString(e)); + } + catch (RuntimeException e) { + synchronized(exceptions) { + exceptions.add(e); } + } } - assertThat(exceptions).isEmpty(); - } + } + }); + blitzer.shutdown(); + + // then + // + if (!exceptions.isEmpty()) { + for (Exception e : exceptions) { + System.err.println(Throwables.getStackTraceAsString(e)); + } + } + assertThat(exceptions).isEmpty(); + } } diff --git a/test/jvm/storm/starter/tools/SlidingWindowCounterTest.java b/test/jvm/storm/starter/tools/SlidingWindowCounterTest.java index 7361ac88e6..1c883e573a 100644 --- a/test/jvm/storm/starter/tools/SlidingWindowCounterTest.java +++ b/test/jvm/storm/starter/tools/SlidingWindowCounterTest.java @@ -1,93 +1,89 @@ package storm.starter.tools; -import static org.fest.assertions.api.Assertions.assertThat; - -import java.util.Map; - import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class SlidingWindowCounterTest { - - private static final int ANY_WINDOW_LENGTH_IN_SLOTS = 2; - private static final Object ANY_OBJECT = "ANY_OBJECT"; - - @DataProvider - public Object[][] illegalWindowLengths() { - return new Object[][] { { -10 }, { -3 }, { -2 }, { -1 }, { 0 }, { 1 } }; - } - - @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalWindowLengths") - public void lessThanTwoSlotsShouldThrowIAE(int windowLengthInSlots) { - new SlidingWindowCounter(windowLengthInSlots); - } - - @DataProvider - public Object[][] legalWindowLengths() { - return new Object[][] { { 2 }, { 3 }, { 20 } }; - } - - @Test(dataProvider = "legalWindowLengths") - public void twoOrMoreSlotsShouldBeValid(int windowLengthInSlots) { - new SlidingWindowCounter(windowLengthInSlots); - } - - @Test - public void newInstanceShouldHaveEmptyCounts() { - // given - SlidingWindowCounter counter = new SlidingWindowCounter(ANY_WINDOW_LENGTH_IN_SLOTS); - - // when - Map counts = counter.getCountsThenAdvanceWindow(); +import java.util.Map; - // then - assertThat(counts).isEmpty(); - } +import static org.fest.assertions.api.Assertions.assertThat; - @DataProvider - public Object[][] simulatedCounterIterations() { - return new Object[][] { - { 2, new int[] { 3, 2, 0, 0, 1, 0, 0, 0 }, new long[] { 3, 5, 2, 0, 1, 1, 0, 0 } }, - { 3, new int[] { 3, 2, 0, 0, 1, 0, 0, 0 }, new long[] { 3, 5, 5, 2, 1, 1, 1, 0 } }, - { 4, new int[] { 3, 2, 0, 0, 1, 0, 0, 0 }, new long[] { 3, 5, 5, 5, 3, 1, 1, 1 } }, - { 5, new int[] { 3, 2, 0, 0, 1, 0, 0, 0 }, new long[] { 3, 5, 5, 5, 6, 3, 1, 1 } }, - { - 5, - new int[] { 3, 11, 5, 13, 7, 17, 0, 3, 50, 600, 7000 }, - new long[] { 3, 14, 19, 32, 39, 53, 42, 40, 77, 670, 7653 } }, }; - } +public class SlidingWindowCounterTest { - @Test(dataProvider = "simulatedCounterIterations") - public void testCounterWithSimulatedRuns(int windowLengthInSlots, int[] incrementsPerIteration, - long[] expCountsPerIteration) { - // given - SlidingWindowCounter counter = new SlidingWindowCounter(windowLengthInSlots); - int numIterations = incrementsPerIteration.length; - - for (int i = 0; i < numIterations; i++) { - int numIncrements = incrementsPerIteration[i]; - long expCounts = expCountsPerIteration[i]; - // Objects are absent if they were zero both this iteration - // and the last -- if only this one, we need to report zero. - boolean expAbsent = ((expCounts == 0) && - ((i == 0) || (expCountsPerIteration[i-1] == 0))); - - // given (for this iteration) - for (int j = 0; j < numIncrements; j++) { - counter.incrementCount(ANY_OBJECT); - } - - // when (for this iteration) - Map counts = counter.getCountsThenAdvanceWindow(); - - // then (for this iteration) - if (expAbsent) { - assertThat(counts).doesNotContainKey(ANY_OBJECT); - } - else { - assertThat(counts.get(ANY_OBJECT)).isEqualTo(expCounts); - } - } + private static final int ANY_WINDOW_LENGTH_IN_SLOTS = 2; + private static final Object ANY_OBJECT = "ANY_OBJECT"; + + @DataProvider + public Object[][] illegalWindowLengths() { + return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 }, { 1 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalWindowLengths") + public void lessThanTwoSlotsShouldThrowIAE(int windowLengthInSlots) { + new SlidingWindowCounter(windowLengthInSlots); + } + + @DataProvider + public Object[][] legalWindowLengths() { + return new Object[][]{ { 2 }, { 3 }, { 20 } }; + } + + @Test(dataProvider = "legalWindowLengths") + public void twoOrMoreSlotsShouldBeValid(int windowLengthInSlots) { + new SlidingWindowCounter(windowLengthInSlots); + } + + @Test + public void newInstanceShouldHaveEmptyCounts() { + // given + SlidingWindowCounter counter = new SlidingWindowCounter(ANY_WINDOW_LENGTH_IN_SLOTS); + + // when + Map counts = counter.getCountsThenAdvanceWindow(); + + // then + assertThat(counts).isEmpty(); + } + + @DataProvider + public Object[][] simulatedCounterIterations() { + return new Object[][]{ { 2, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 2, 0, 1, 1, 0, 0 } }, + { 3, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 5, 2, 1, 1, 1, 0 } }, + { 4, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 5, 5, 3, 1, 1, 1 } }, + { 5, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 5, 5, 6, 3, 1, 1 } }, + { 5, new int[]{ 3, 11, 5, 13, 7, 17, 0, 3, 50, 600, 7000 }, + new long[]{ 3, 14, 19, 32, 39, 53, 42, 40, 77, 670, 7653 } }, }; + } + + @Test(dataProvider = "simulatedCounterIterations") + public void testCounterWithSimulatedRuns(int windowLengthInSlots, int[] incrementsPerIteration, + long[] expCountsPerIteration) { + // given + SlidingWindowCounter counter = new SlidingWindowCounter(windowLengthInSlots); + int numIterations = incrementsPerIteration.length; + + for (int i = 0; i < numIterations; i++) { + int numIncrements = incrementsPerIteration[i]; + long expCounts = expCountsPerIteration[i]; + // Objects are absent if they were zero both this iteration + // and the last -- if only this one, we need to report zero. + boolean expAbsent = ((expCounts == 0) && ((i == 0) || (expCountsPerIteration[i - 1] == 0))); + + // given (for this iteration) + for (int j = 0; j < numIncrements; j++) { + counter.incrementCount(ANY_OBJECT); + } + + // when (for this iteration) + Map counts = counter.getCountsThenAdvanceWindow(); + + // then (for this iteration) + if (expAbsent) { + assertThat(counts).doesNotContainKey(ANY_OBJECT); + } + else { + assertThat(counts.get(ANY_OBJECT)).isEqualTo(expCounts); + } } + } } diff --git a/test/jvm/storm/starter/tools/SlotBasedCounterTest.java b/test/jvm/storm/starter/tools/SlotBasedCounterTest.java index be174d439d..aeeec63407 100644 --- a/test/jvm/storm/starter/tools/SlotBasedCounterTest.java +++ b/test/jvm/storm/starter/tools/SlotBasedCounterTest.java @@ -1,164 +1,164 @@ package storm.starter.tools; -import static org.fest.assertions.api.Assertions.assertThat; - -import java.util.Map; - import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class SlotBasedCounterTest { - - private static final int ANY_NUM_SLOTS = 1; - private static final int ANY_SLOT = 0; - private static final Object ANY_OBJECT = "ANY_OBJECT"; - - @DataProvider - public Object[][] illegalNumSlotsData() { - return new Object[][] { { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; - } - - @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalNumSlotsData") - public void negativeOrZeroNumSlotsShouldThrowIAE(int numSlots) { - new SlotBasedCounter(numSlots); - } - - @DataProvider - public Object[][] legalNumSlotsData() { - return new Object[][] { { 1 }, { 2 }, { 3 }, { 20 } }; - } - - @Test(dataProvider = "legalNumSlotsData") - public void positiveNumSlotsShouldBeOk(int numSlots) { - new SlotBasedCounter(numSlots); - } - - @Test - public void newInstanceShouldHaveEmptyCounts() { - // given - SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); - - // when - Map counts = counter.getCounts(); - - // then - assertThat(counts).isEmpty(); - } - - @Test - public void shouldReturnNonEmptyCountsWhenAtLeastOneObjectWasCounted() { - // given - SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); - counter.incrementCount(ANY_OBJECT, ANY_SLOT); - - // when - Map counts = counter.getCounts(); - - // then - assertThat(counts).isNotEmpty(); - - // additional tests that go beyond what this test is primarily about - assertThat(counts.size()).isEqualTo(1); - assertThat(counts.get(ANY_OBJECT)).isEqualTo(1); - } - - @DataProvider - public Object[][] incrementCountData() { - return new Object[][] { { new String[] { "foo", "bar" }, new int[] { 3, 2 } } }; - } - - @Test(dataProvider = "incrementCountData") - public void shouldIncrementCount(Object[] objects, int[] expCounts) { - // given - SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); - - // when - for (int i = 0; i < objects.length; i++) { - Object obj = objects[i]; - int numIncrements = expCounts[i]; - for (int j = 0; j < numIncrements; j++) { - counter.incrementCount(obj, ANY_SLOT); - } - } - - // then - for (int i = 0; i < objects.length; i++) { - assertThat(counter.getCount(objects[i], ANY_SLOT)).isEqualTo(expCounts[i]); - } - assertThat(counter.getCount("nonexistentObject", ANY_SLOT)).isEqualTo(0); - } - - @Test - public void shouldReturnZeroForNonexistentObject() { - // given - SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); - - // when - counter.incrementCount("somethingElse", ANY_SLOT); +import java.util.Map; - // then - assertThat(counter.getCount("nonexistentObject", ANY_SLOT)).isEqualTo(0); - } +import static org.fest.assertions.api.Assertions.assertThat; - @Test - public void shouldIncrementCountOnlyOneSlotAtATime() { - // given - int numSlots = 3; - Object obj = Long.valueOf(10); - SlotBasedCounter counter = new SlotBasedCounter(numSlots); - - // when (empty) - // then - assertThat(counter.getCount(obj, 0)).isEqualTo(0); - assertThat(counter.getCount(obj, 1)).isEqualTo(0); - assertThat(counter.getCount(obj, 2)).isEqualTo(0); - - // when - counter.incrementCount(obj, 1); - - // then - assertThat(counter.getCount(obj, 0)).isEqualTo(0); - assertThat(counter.getCount(obj, 1)).isEqualTo(1); - assertThat(counter.getCount(obj, 2)).isEqualTo(0); - } +public class SlotBasedCounterTest { - @Test - public void wipeSlotShouldSetAllCountsInSlotToZero() { - // given - SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); - Object countWasOne = "countWasOne"; - Object countWasThree = "countWasThree"; - counter.incrementCount(countWasOne, ANY_SLOT); - counter.incrementCount(countWasThree, ANY_SLOT); - counter.incrementCount(countWasThree, ANY_SLOT); - counter.incrementCount(countWasThree, ANY_SLOT); - - // when - counter.wipeSlot(ANY_SLOT); - - // then - assertThat(counter.getCount(countWasOne, ANY_SLOT)).isEqualTo(0); - assertThat(counter.getCount(countWasThree, ANY_SLOT)).isEqualTo(0); + private static final int ANY_NUM_SLOTS = 1; + private static final int ANY_SLOT = 0; + private static final Object ANY_OBJECT = "ANY_OBJECT"; + + @DataProvider + public Object[][] illegalNumSlotsData() { + return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; + } + + @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalNumSlotsData") + public void negativeOrZeroNumSlotsShouldThrowIAE(int numSlots) { + new SlotBasedCounter(numSlots); + } + + @DataProvider + public Object[][] legalNumSlotsData() { + return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } }; + } + + @Test(dataProvider = "legalNumSlotsData") + public void positiveNumSlotsShouldBeOk(int numSlots) { + new SlotBasedCounter(numSlots); + } + + @Test + public void newInstanceShouldHaveEmptyCounts() { + // given + SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); + + // when + Map counts = counter.getCounts(); + + // then + assertThat(counts).isEmpty(); + } + + @Test + public void shouldReturnNonEmptyCountsWhenAtLeastOneObjectWasCounted() { + // given + SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); + counter.incrementCount(ANY_OBJECT, ANY_SLOT); + + // when + Map counts = counter.getCounts(); + + // then + assertThat(counts).isNotEmpty(); + + // additional tests that go beyond what this test is primarily about + assertThat(counts.size()).isEqualTo(1); + assertThat(counts.get(ANY_OBJECT)).isEqualTo(1); + } + + @DataProvider + public Object[][] incrementCountData() { + return new Object[][]{ { new String[]{ "foo", "bar" }, new int[]{ 3, 2 } } }; + } + + @Test(dataProvider = "incrementCountData") + public void shouldIncrementCount(Object[] objects, int[] expCounts) { + // given + SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); + + // when + for (int i = 0; i < objects.length; i++) { + Object obj = objects[i]; + int numIncrements = expCounts[i]; + for (int j = 0; j < numIncrements; j++) { + counter.incrementCount(obj, ANY_SLOT); + } } - @Test - public void wipeZerosShouldRemoveAnyObjectsWithZeroTotalCount() { - // given - SlotBasedCounter counter = new SlotBasedCounter(2); - int wipeSlot = 0; - int otherSlot = 1; - Object willBeRemoved = "willBeRemoved"; - Object willContinueToBeTracked = "willContinueToBeTracked"; - counter.incrementCount(willBeRemoved, wipeSlot); - counter.incrementCount(willContinueToBeTracked, wipeSlot); - counter.incrementCount(willContinueToBeTracked, otherSlot); - - // when - counter.wipeSlot(wipeSlot); - counter.wipeZeros(); - - // then - assertThat(counter.getCounts()).doesNotContainKey(willBeRemoved); - assertThat(counter.getCounts()).containsKey(willContinueToBeTracked); + // then + for (int i = 0; i < objects.length; i++) { + assertThat(counter.getCount(objects[i], ANY_SLOT)).isEqualTo(expCounts[i]); } + assertThat(counter.getCount("nonexistentObject", ANY_SLOT)).isEqualTo(0); + } + + @Test + public void shouldReturnZeroForNonexistentObject() { + // given + SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); + + // when + counter.incrementCount("somethingElse", ANY_SLOT); + + // then + assertThat(counter.getCount("nonexistentObject", ANY_SLOT)).isEqualTo(0); + } + + @Test + public void shouldIncrementCountOnlyOneSlotAtATime() { + // given + int numSlots = 3; + Object obj = Long.valueOf(10); + SlotBasedCounter counter = new SlotBasedCounter(numSlots); + + // when (empty) + // then + assertThat(counter.getCount(obj, 0)).isEqualTo(0); + assertThat(counter.getCount(obj, 1)).isEqualTo(0); + assertThat(counter.getCount(obj, 2)).isEqualTo(0); + + // when + counter.incrementCount(obj, 1); + + // then + assertThat(counter.getCount(obj, 0)).isEqualTo(0); + assertThat(counter.getCount(obj, 1)).isEqualTo(1); + assertThat(counter.getCount(obj, 2)).isEqualTo(0); + } + + @Test + public void wipeSlotShouldSetAllCountsInSlotToZero() { + // given + SlotBasedCounter counter = new SlotBasedCounter(ANY_NUM_SLOTS); + Object countWasOne = "countWasOne"; + Object countWasThree = "countWasThree"; + counter.incrementCount(countWasOne, ANY_SLOT); + counter.incrementCount(countWasThree, ANY_SLOT); + counter.incrementCount(countWasThree, ANY_SLOT); + counter.incrementCount(countWasThree, ANY_SLOT); + + // when + counter.wipeSlot(ANY_SLOT); + + // then + assertThat(counter.getCount(countWasOne, ANY_SLOT)).isEqualTo(0); + assertThat(counter.getCount(countWasThree, ANY_SLOT)).isEqualTo(0); + } + + @Test + public void wipeZerosShouldRemoveAnyObjectsWithZeroTotalCount() { + // given + SlotBasedCounter counter = new SlotBasedCounter(2); + int wipeSlot = 0; + int otherSlot = 1; + Object willBeRemoved = "willBeRemoved"; + Object willContinueToBeTracked = "willContinueToBeTracked"; + counter.incrementCount(willBeRemoved, wipeSlot); + counter.incrementCount(willContinueToBeTracked, wipeSlot); + counter.incrementCount(willContinueToBeTracked, otherSlot); + + // when + counter.wipeSlot(wipeSlot); + counter.wipeZeros(); + + // then + assertThat(counter.getCounts()).doesNotContainKey(willBeRemoved); + assertThat(counter.getCounts()).containsKey(willContinueToBeTracked); + } } From 6d56eae9f0707462bdb3502b19debaceb4e7f266 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 12:45:19 +0200 Subject: [PATCH 213/349] Rankings: add pruneZeroCounts() method --- src/jvm/storm/starter/tools/Rankings.java | 17 ++++++++++- .../jvm/storm/starter/tools/RankingsTest.java | 28 ++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/jvm/storm/starter/tools/Rankings.java b/src/jvm/storm/starter/tools/Rankings.java index a3f421a5fe..8dbb35f151 100644 --- a/src/jvm/storm/starter/tools/Rankings.java +++ b/src/jvm/storm/starter/tools/Rankings.java @@ -89,7 +89,22 @@ private void shrinkRankingsIfNeeded() { } } + /** + * Removes ranking entries that have a count of zero. + */ + public void pruneZeroCounts() { + int i = 0; + while (i < rankedItems.size()) { + if (rankedItems.get(i).getCount() == 0) { + rankedItems.remove(i); + } + else { + i++; + } + } + } + public String toString() { return rankedItems.toString(); } -} +} \ No newline at end of file diff --git a/test/jvm/storm/starter/tools/RankingsTest.java b/test/jvm/storm/starter/tools/RankingsTest.java index 138a8b33be..7d3b4da490 100644 --- a/test/jvm/storm/starter/tools/RankingsTest.java +++ b/test/jvm/storm/starter/tools/RankingsTest.java @@ -15,6 +15,7 @@ public class RankingsTest { private static final int ANY_TOPN = 42; private static final Rankable ANY_RANKABLE = new RankableObjectWithFields("someObject", ANY_TOPN); + private static final Rankable ZERO = new RankableObjectWithFields("ZERO_COUNT", 0); private static final Rankable A = new RankableObjectWithFields("A", 1); private static final Rankable B = new RankableObjectWithFields("B", 2); private static final Rankable C = new RankableObjectWithFields("C", 3); @@ -194,6 +195,31 @@ public void shouldNotRankDuplicateObjectsMoreThanOnce(List duplicates) assertThat(rankings.size()).isEqualTo(1); } + @DataProvider + public Object[][] removeZeroRankingsData() { + return new Object[][]{ { Lists.newArrayList(A, ZERO), Lists.newArrayList(A) }, { Lists.newArrayList(A), + Lists.newArrayList(A) }, { Lists.newArrayList(ZERO, A), Lists.newArrayList(A) }, { Lists.newArrayList(ZERO), + Lists.newArrayList() }, { Lists.newArrayList(ZERO, new RankableObjectWithFields("ZERO2", 0)), + Lists.newArrayList() }, { Lists.newArrayList(B, ZERO, new RankableObjectWithFields("ZERO2", 0), D, + new RankableObjectWithFields("ZERO3", 0), new RankableObjectWithFields("ZERO4", 0), C), Lists.newArrayList(D, C, + B) }, { Lists.newArrayList(A, ZERO, B), Lists.newArrayList(B, A) } }; + } + + @Test(dataProvider = "removeZeroRankingsData") + public void shouldRemoveZeroCounts(List unsorted, List expSorted) { + // given + Rankings rankings = new Rankings(unsorted.size()); + for (Rankable r : unsorted) { + rankings.updateWith(r); + } + + // when + rankings.pruneZeroCounts(); + + // then + assertThat(rankings.getRankings()).isEqualTo(expSorted); + } + @Test public void updatingWithNewRankablesShouldBeThreadSafe() throws InterruptedException { // given @@ -232,4 +258,4 @@ public void run() { } assertThat(exceptions).isEmpty(); } -} +} \ No newline at end of file From eca746003408107dcfff4846f6bde55e33ffacc8 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 12:45:50 +0200 Subject: [PATCH 214/349] Rankings: use private helper method to implement getRankings() for cleaner code --- src/jvm/storm/starter/tools/Rankings.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/jvm/storm/starter/tools/Rankings.java b/src/jvm/storm/starter/tools/Rankings.java index 8dbb35f151..5ff471a75d 100644 --- a/src/jvm/storm/starter/tools/Rankings.java +++ b/src/jvm/storm/starter/tools/Rankings.java @@ -40,6 +40,10 @@ public int size() { } public List getRankings() { + return defensiveCopyOf(rankedItems); + } + + private List defensiveCopyOf(List list) { return Lists.newArrayList(rankedItems); } From fa424aefbed93f5dbddc9ab38b2c5ddc85c592a0 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 12:52:03 +0200 Subject: [PATCH 215/349] project.clj: add explicit :test-paths setting so that Java IDE's can properly match Java package names and directory names for test classes --- project.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/project.clj b/project.clj index 21c31ec869..89192751e8 100644 --- a/project.clj +++ b/project.clj @@ -1,6 +1,7 @@ (defproject storm-starter "0.0.1-SNAPSHOT" :source-paths ["src/clj"] :java-source-paths ["src/jvm"] + :test-paths ["test/jvm"] :resource-paths ["multilang"] :aot :all :repositories { From d596aaf2c14064a39865c177aea9bbeca20d2493 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 12:55:09 +0200 Subject: [PATCH 216/349] TotalRankingsBolt: prune zero count rankings after each emit --- src/jvm/storm/starter/bolt/TotalRankingsBolt.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/jvm/storm/starter/bolt/TotalRankingsBolt.java b/src/jvm/storm/starter/bolt/TotalRankingsBolt.java index 846ff493e5..217443e4c2 100644 --- a/src/jvm/storm/starter/bolt/TotalRankingsBolt.java +++ b/src/jvm/storm/starter/bolt/TotalRankingsBolt.java @@ -31,6 +31,7 @@ public TotalRankingsBolt(int topN, int emitFrequencyInSeconds) { void updateRankingsWithTuple(Tuple tuple) { Rankings rankingsToBeMerged = (Rankings) tuple.getValue(0); super.getRankings().updateWith(rankingsToBeMerged); + super.getRankings().pruneZeroCounts(); } @Override From d5ec8f40206cc58dd0c5ecf4355a18f637abde22 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 12:56:29 +0200 Subject: [PATCH 217/349] RankableObjectWithFields: update Java docs --- src/jvm/storm/starter/tools/RankableObjectWithFields.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jvm/storm/starter/tools/RankableObjectWithFields.java b/src/jvm/storm/starter/tools/RankableObjectWithFields.java index 1fdad374ce..3b27d0d5dc 100644 --- a/src/jvm/storm/starter/tools/RankableObjectWithFields.java +++ b/src/jvm/storm/starter/tools/RankableObjectWithFields.java @@ -43,7 +43,7 @@ public RankableObjectWithFields(Object obj, long count, Object... otherFields) { * * @param tuple * - * @return + * @return new instance based on the provided tuple */ public static RankableObjectWithFields from(Tuple tuple) { List otherFields = Lists.newArrayList(tuple.getValues()); From 8b4012082975fd7b31505ebafbcb9404c55369ae Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Thu, 29 Aug 2013 12:57:10 +0200 Subject: [PATCH 218/349] SlidingWindowCounter: update Java docs --- src/jvm/storm/starter/tools/SlidingWindowCounter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jvm/storm/starter/tools/SlidingWindowCounter.java b/src/jvm/storm/starter/tools/SlidingWindowCounter.java index dff6a593f8..38becdca8f 100644 --- a/src/jvm/storm/starter/tools/SlidingWindowCounter.java +++ b/src/jvm/storm/starter/tools/SlidingWindowCounter.java @@ -80,7 +80,7 @@ public void incrementCount(T obj) { * successfully processed "upstream" (i.e. by the caller). Knowing this we will start counting any subsequent * objects within the next "chunk" of the sliding window. * - * @return + * @return The current (total) counts of all tracked objects. */ public Map getCountsThenAdvanceWindow() { Map counts = objCounter.getCounts(); From 1a19f3ddaca0d2ab2c388d70d03d7ebd63590879 Mon Sep 17 00:00:00 2001 From: John Gilmore Date: Tue, 1 Oct 2013 10:59:47 +0200 Subject: [PATCH 219/349] Shell component serialisation is now plugable with a default JSON serialiser being used. Moved multilang classes to multilang directory. --- project.clj | 2 +- .../src/clj/backtype/storm/bootstrap.clj | 5 +- .../backtype/storm/clojure/RichShellBolt.java | 2 +- .../storm/clojure/RichShellSpout.java | 2 +- .../backtype/storm/multilang/Emission.java | 69 +++++++++ .../backtype/storm/multilang/ISerializer.java | 19 +++ .../backtype/storm/multilang/Immission.java | 42 +++++ .../storm/multilang/JsonSerializer.java | 144 ++++++++++++++++++ .../storm/multilang/NoOutputException.java | 8 + .../storm/{task => multilang}/ShellBolt.java | 128 +++++++--------- .../storm/multilang/ShellProcess.java | 101 ++++++++++++ .../backtype/storm/multilang/ShellSpout.java | 115 ++++++++++++++ .../backtype/storm/multilang/SpoutMsg.java | 18 +++ .../jvm/backtype/storm/spout/ShellSpout.java | 120 --------------- .../backtype/storm/utils/ShellProcess.java | 132 ---------------- 15 files changed, 578 insertions(+), 329 deletions(-) create mode 100644 storm-core/src/jvm/backtype/storm/multilang/Emission.java create mode 100644 storm-core/src/jvm/backtype/storm/multilang/ISerializer.java create mode 100644 storm-core/src/jvm/backtype/storm/multilang/Immission.java create mode 100644 storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java create mode 100644 storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java rename storm-core/src/jvm/backtype/storm/{task => multilang}/ShellBolt.java (59%) create mode 100644 storm-core/src/jvm/backtype/storm/multilang/ShellProcess.java create mode 100644 storm-core/src/jvm/backtype/storm/multilang/ShellSpout.java create mode 100644 storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java delete mode 100644 storm-core/src/jvm/backtype/storm/spout/ShellSpout.java delete mode 100644 storm-core/src/jvm/backtype/storm/utils/ShellProcess.java diff --git a/project.clj b/project.clj index ed1fdbc3f1..f913924b0c 100644 --- a/project.clj +++ b/project.clj @@ -10,7 +10,7 @@ :archive "https://groups.google.com/group/storm-user" :post "storm-user@googlegroups.com"} :dependencies [~@DEPENDENCIES] - :plugins [[~'lein-sub "0.2.1"]] + :plugins [[~'lein-sub "0.2.1"],[no-man-is-an-island/lein-eclipse "2.0.0"]] :min-lein-version "2.0.0" :sub [~@MODULES] )) diff --git a/storm-core/src/clj/backtype/storm/bootstrap.clj b/storm-core/src/clj/backtype/storm/bootstrap.clj index 70b80057ea..1075c664df 100644 --- a/storm-core/src/clj/backtype/storm/bootstrap.clj +++ b/storm-core/src/clj/backtype/storm/bootstrap.clj @@ -12,10 +12,11 @@ RegisteredGlobalState ThriftTopologyUtils DisruptorQueue MutableObject MutableLong])) (import (quote [backtype.storm.serialization KryoTupleSerializer KryoTupleDeserializer])) - (import (quote [backtype.storm.spout ISpout SpoutOutputCollector ISpoutOutputCollector ShellSpout])) + (import (quote [backtype.storm.spout ISpout SpoutOutputCollector ISpoutOutputCollector])) + (import (quote [backtype.storm.multilang ShellSpout ShellBolt])) (import (quote [backtype.storm.tuple Tuple TupleImpl Fields MessageId])) (import (quote [backtype.storm.task IBolt IOutputCollector - OutputCollector TopologyContext ShellBolt + OutputCollector TopologyContext GeneralTopologyContext WorkerTopologyContext])) (import (quote [backtype.storm.coordination CoordinatedBolt CoordinatedBolt$SourceArgs IBatchBolt BatchBoltExecutor])) diff --git a/storm-core/src/jvm/backtype/storm/clojure/RichShellBolt.java b/storm-core/src/jvm/backtype/storm/clojure/RichShellBolt.java index 6be104ea1f..6ff669eaf4 100644 --- a/storm-core/src/jvm/backtype/storm/clojure/RichShellBolt.java +++ b/storm-core/src/jvm/backtype/storm/clojure/RichShellBolt.java @@ -1,7 +1,7 @@ package backtype.storm.clojure; import backtype.storm.generated.StreamInfo; -import backtype.storm.task.ShellBolt; +import backtype.storm.multilang.ShellBolt; import backtype.storm.topology.IRichBolt; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.tuple.Fields; diff --git a/storm-core/src/jvm/backtype/storm/clojure/RichShellSpout.java b/storm-core/src/jvm/backtype/storm/clojure/RichShellSpout.java index cb5947f692..57410e1e9c 100644 --- a/storm-core/src/jvm/backtype/storm/clojure/RichShellSpout.java +++ b/storm-core/src/jvm/backtype/storm/clojure/RichShellSpout.java @@ -1,7 +1,7 @@ package backtype.storm.clojure; import backtype.storm.generated.StreamInfo; -import backtype.storm.spout.ShellSpout; +import backtype.storm.multilang.ShellSpout; import backtype.storm.topology.IRichSpout; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.tuple.Fields; diff --git a/storm-core/src/jvm/backtype/storm/multilang/Emission.java b/storm-core/src/jvm/backtype/storm/multilang/Emission.java new file mode 100644 index 0000000000..10544580be --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/multilang/Emission.java @@ -0,0 +1,69 @@ +package backtype.storm.multilang; + +import java.util.ArrayList; +import java.util.List; + +public class Emission { + private String command; + private String id; + private List anchors; + private String stream; + private long task; + private String msg; + private List tuple; + + public String getCommand() { + return command; + } + public void setCommand(String command) { + this.command = command; + } + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + public List getAnchors() { + return anchors; + } + public void setAnchors(List anchors) { + this.anchors = anchors; + } + public void addAnchor(String anchor) { + if (anchors == null) { + anchors = new ArrayList(); + } + this.anchors.add(anchor); + } + public String getStream() { + return stream; + } + public void setStream(String stream) { + this.stream = stream; + } + public long getTask() { + return task; + } + public void setTask(long task) { + this.task = task; + } + public String getMsg() { + return msg; + } + public void setMsg(String msg) { + this.msg = msg; + } + public List getTuple() { + return tuple; + } + public void setTuple(List tuple) { + this.tuple = tuple; + } + public void addTuple(Object tuple) { + if (this.tuple == null) { + this.tuple = new ArrayList(); + } + this.tuple.add(tuple); + } +} diff --git a/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java b/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java new file mode 100644 index 0000000000..61a28a930e --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java @@ -0,0 +1,19 @@ +package backtype.storm.multilang; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +import backtype.storm.task.TopologyContext; + +public interface ISerializer extends Serializable { + void initialize (OutputStream processIn, InputStream processOut); + Number connect (Map conf, TopologyContext context) throws IOException, NoOutputException; + Emission readEmission() throws IOException, NoOutputException; + void writeImmission(Immission immission) throws IOException; + void writeSpoutMsg(SpoutMsg msg) throws IOException; + void writeTaskIds(List taskIds) throws IOException; +} diff --git a/storm-core/src/jvm/backtype/storm/multilang/Immission.java b/storm-core/src/jvm/backtype/storm/multilang/Immission.java new file mode 100644 index 0000000000..c20df53506 --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/multilang/Immission.java @@ -0,0 +1,42 @@ +package backtype.storm.multilang; + +import java.util.List; + +public class Immission { + private String id; + private String comp; + private String stream; + private long task; + private List tuple; + + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + public String getComp() { + return comp; + } + public void setComp(String comp) { + this.comp = comp; + } + public String getStream() { + return stream; + } + public void setStream(String stream) { + this.stream = stream; + } + public long getTask() { + return task; + } + public void setTask(long task) { + this.task = task; + } + public List getTuple() { + return tuple; + } + public void setTuple(List tuple) { + this.tuple = tuple; + } +} diff --git a/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java b/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java new file mode 100644 index 0000000000..88eead29ad --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java @@ -0,0 +1,144 @@ +package backtype.storm.multilang; + +import java.io.BufferedReader; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; + +import backtype.storm.task.TopologyContext; +import backtype.storm.tuple.Tuple; +import backtype.storm.utils.Utils; + +public class JsonSerializer implements ISerializer { + private DataOutputStream processIn; + private BufferedReader processOut; + + public void initialize (OutputStream processIn, InputStream processOut) { + this.processIn = new DataOutputStream(processIn);; + this.processOut = new BufferedReader(new InputStreamReader(processOut)); + } + + public Number connect (Map conf, TopologyContext context) throws IOException, NoOutputException { + JSONObject setupInfo = new JSONObject(); + setupInfo.put("pidDir", context.getPIDDir()); + setupInfo.put("conf", conf); + setupInfo.put("context", context); + writeMessage(setupInfo); + + Number pid = (Number)((JSONObject)readMessage()).get("pid"); + return pid; + } + + public void writeImmission(Immission immission) throws IOException { + JSONObject obj = new JSONObject(); + obj.put("id", immission.getId()); + obj.put("comp", immission.getComp()); + obj.put("stream", immission.getStream()); + obj.put("task", immission.getTask()); + obj.put("tuple", immission.getTuple()); + writeMessage(obj); + } + + public void writeSpoutMsg(SpoutMsg msg) throws IOException { + JSONObject obj = new JSONObject(); + obj.put("command", msg.getCommand()); + obj.put("id", msg.getId()); + writeMessage(obj); + } + + public void writeTaskIds(List taskIds) throws IOException { + writeMessage(taskIds); + } + + private void writeMessage(Object msg) throws IOException { + writeString(JSONValue.toJSONString(msg)); + } + + private void writeString(String str) throws IOException { + byte[] strBytes = str.getBytes("UTF-8"); + processIn.write(strBytes, 0, strBytes.length); + processIn.writeBytes("\nend\n"); + processIn.flush(); + } + + public Emission readEmission() throws IOException, NoOutputException { + JSONObject msg = (JSONObject)readMessage(); + Emission emission = new Emission(); + + String command = (String) msg.get("command"); + emission.setCommand(command); + + String stream = (String) msg.get("stream"); + if(stream == null) stream = Utils.DEFAULT_STREAM_ID; + emission.setStream(stream); + + Object taskObj = msg.get("task"); + if (taskObj != null) { + emission.setTask((Long) taskObj); + } + + emission.setTuple((List) msg.get("tuple")); + + List anchors = new ArrayList(); + Object anchorObj = msg.get("anchors"); + if(anchorObj!=null) { + if(anchorObj instanceof String) { + anchorObj = Arrays.asList(anchorObj); + } + for(Object o: (List) anchorObj) { + emission.addAnchor((String)o); + } + } + + return emission; + } + + private Object readMessage() throws IOException, NoOutputException { + String string = readString(); + Object msg = JSONValue.parse(string); + if (msg != null) { + return msg; + } else { + throw new IOException("unable to parse: " + string); + } + } + + private String readString() throws IOException, NoOutputException { + StringBuilder line = new StringBuilder(); + + //synchronized (processOut) { + while (true) { + String subline = processOut.readLine(); + if(subline==null) { + StringBuilder errorMessage = new StringBuilder(); + errorMessage.append("Pipe to subprocess seems to be broken!"); + if (line.length() == 0) { + errorMessage.append(" No output read.\n"); + } + else { + errorMessage.append(" Currently read output: " + line.toString() + "\n"); + } + errorMessage.append("Serializer Exception:\n"); + throw new NoOutputException(errorMessage.toString()); + } + if(subline.equals("end")) { + break; + } + if(line.length()!=0) { + line.append("\n"); + } + line.append(subline); + } + //} + return line.toString(); + } +} diff --git a/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java b/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java new file mode 100644 index 0000000000..595a0ae7e2 --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java @@ -0,0 +1,8 @@ +package backtype.storm.multilang; + +public class NoOutputException extends Exception { + public NoOutputException() { super(); } + public NoOutputException(String message) { super(message); } + public NoOutputException(String message, Throwable cause) { super(message, cause); } + public NoOutputException(Throwable cause) { super(cause); } +} \ No newline at end of file diff --git a/storm-core/src/jvm/backtype/storm/task/ShellBolt.java b/storm-core/src/jvm/backtype/storm/multilang/ShellBolt.java similarity index 59% rename from storm-core/src/jvm/backtype/storm/task/ShellBolt.java rename to storm-core/src/jvm/backtype/storm/multilang/ShellBolt.java index 854aa8f532..acd60c9809 100644 --- a/storm-core/src/jvm/backtype/storm/task/ShellBolt.java +++ b/storm-core/src/jvm/backtype/storm/multilang/ShellBolt.java @@ -1,22 +1,20 @@ -package backtype.storm.task; +package backtype.storm.multilang; import backtype.storm.generated.ShellComponent; -import backtype.storm.tuple.MessageId; +import backtype.storm.task.IBolt; +import backtype.storm.task.OutputCollector; +import backtype.storm.task.TopologyContext; import backtype.storm.tuple.Tuple; -import backtype.storm.utils.Utils; -import backtype.storm.utils.ShellProcess; -import java.io.IOException; +import backtype.storm.multilang.ShellProcess; import java.util.ArrayList; -import java.util.Arrays; +import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; import static java.util.concurrent.TimeUnit.SECONDS; -import java.util.List; import java.util.Map; import java.util.Random; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.json.simple.JSONObject; /** * A bolt that shells out to another process to process tuples. ShellBolt @@ -57,54 +55,53 @@ public class ShellBolt implements IBolt { private volatile Throwable _exception; private LinkedBlockingQueue _pendingWrites = new LinkedBlockingQueue(); private Random _rand; - + private Thread _readerThread; private Thread _writerThread; public ShellBolt(ShellComponent component) { this(component.get_execution_command(), component.get_script()); + _process = new ShellProcess(new JsonSerializer(), _command); } public ShellBolt(String... command) { _command = command; + _process = new ShellProcess(new JsonSerializer(), _command); + } + + public ShellBolt(ISerializer serializer, String... command) { + _command = command; + _process = new ShellProcess(serializer, _command); } public void prepare(Map stormConf, TopologyContext context, final OutputCollector collector) { _rand = new Random(); - _process = new ShellProcess(_command); _collector = collector; - try { - //subprocesses must send their pid first thing - Number subpid = _process.launch(stormConf, context); - LOG.info("Launched subprocess with pid " + subpid); - } catch (IOException e) { - throw new RuntimeException("Error when launching multilang subprocess\n" + _process.getErrorsString(), e); - } + //subprocesses must send their pid first thing + Number subpid = _process.launch(stormConf, context); + LOG.info("Launched subprocess with pid " + subpid); // reader _readerThread = new Thread(new Runnable() { public void run() { while (_running) { try { - JSONObject action = _process.readMessage(); - if (action == null) { - // ignore sync - } + Emission emission = _process.readEmission(); - String command = (String) action.get("command"); + String command = emission.getCommand(); if(command.equals("ack")) { - handleAck(action); + handleAck(emission.getId()); } else if (command.equals("fail")) { - handleFail(action); + handleFail(emission.getId()); } else if (command.equals("error")) { - handleError(action); + handleError(emission.getMsg()); } else if (command.equals("log")) { - String msg = (String) action.get("msg"); + String msg = emission.getMsg(); LOG.info("Shell msg: " + msg); } else if (command.equals("emit")) { - handleEmit(action); + handleEmit(emission); } } catch (InterruptedException e) { } catch (Throwable t) { @@ -113,19 +110,20 @@ public void run() { } } }); - + _readerThread.start(); _writerThread = new Thread(new Runnable() { public void run() { while (_running) { try { + // FIXME: This can either be TaskIds or Immissions Object write = _pendingWrites.poll(1, SECONDS); - if (write != null) { - _process.writeMessage(write); + if (write instanceof Immission) { + _process.writeImmission((Immission)write); + } else if (write instanceof List) { + } - // drain the error stream to avoid dead lock because of full error stream buffer - _process.drainErrorStream(); } catch (InterruptedException e) { } catch (Throwable t) { die(t); @@ -133,7 +131,7 @@ public void run() { } } }); - + _writerThread.start(); } @@ -146,13 +144,14 @@ public void execute(Tuple input) { String genId = Long.toString(_rand.nextLong()); _inputs.put(genId, input); try { - JSONObject obj = new JSONObject(); - obj.put("id", genId); - obj.put("comp", input.getSourceComponent()); - obj.put("stream", input.getSourceStreamId()); - obj.put("task", input.getSourceTask()); - obj.put("tuple", input.getValues()); - _pendingWrites.put(obj); + Immission immission = new Immission(); + immission.setId(genId); + immission.setComp(input.getSourceComponent()); + immission.setStream(input.getSourceStreamId()); + immission.setTask(input.getSourceTask()); + immission.setTuple(input.getValues()); + + _pendingWrites.put(immission); } catch(InterruptedException e) { throw new RuntimeException("Error during multilang processing", e); } @@ -164,8 +163,7 @@ public void cleanup() { _inputs.clear(); } - private void handleAck(Map action) { - String id = (String) action.get("id"); + private void handleAck(String id) { Tuple acked = _inputs.remove(id); if(acked==null) { throw new RuntimeException("Acked a non-existent or already acked/failed id: " + id); @@ -173,8 +171,7 @@ private void handleAck(Map action) { _collector.ack(acked); } - private void handleFail(Map action) { - String id = (String) action.get("id"); + private void handleFail(String id) { Tuple failed = _inputs.remove(id); if(failed==null) { throw new RuntimeException("Failed a non-existent or already acked/failed id: " + id); @@ -182,38 +179,25 @@ private void handleFail(Map action) { _collector.fail(failed); } - private void handleError(Map action) { - String msg = (String) action.get("msg"); + private void handleError(String msg) { _collector.reportError(new Exception("Shell Process Exception: " + msg)); } - private void handleEmit(Map action) throws InterruptedException { - String stream = (String) action.get("stream"); - if(stream==null) stream = Utils.DEFAULT_STREAM_ID; - Long task = (Long) action.get("task"); - List tuple = (List) action.get("tuple"); - List anchors = new ArrayList(); - Object anchorObj = action.get("anchors"); - if(anchorObj!=null) { - if(anchorObj instanceof String) { - anchorObj = Arrays.asList(anchorObj); - } - for(Object o: (List) anchorObj) { - Tuple t = _inputs.get((String) o); - if (t == null) { - throw new RuntimeException("Anchored onto " + o + " after ack/fail"); - } - anchors.add(t); - } - } - if(task==null) { - List outtasks = _collector.emit(stream, anchors, tuple); - Object need_task_ids = action.get("need_task_ids"); - if (need_task_ids == null || ((Boolean) need_task_ids).booleanValue()) { - _pendingWrites.put(outtasks); - } + private void handleEmit(Emission emission) throws InterruptedException { + List anchors = new ArrayList(); + for (String anchor : emission.getAnchors()) { + Tuple t = _inputs.get(anchor); + if (t == null) { + throw new RuntimeException("Anchored onto " + anchor + " after ack/fail"); + } + anchors.add(t); + } + + if(emission.getTask() == 0) { + List outtasks = _collector.emit(emission.getStream(), anchors, emission.getTuple()); + _pendingWrites.put(outtasks); } else { - _collector.emitDirect((int)task.longValue(), stream, anchors, tuple); + _collector.emitDirect((int)emission.getTask(), emission.getStream(), anchors, emission.getTuple()); } } diff --git a/storm-core/src/jvm/backtype/storm/multilang/ShellProcess.java b/storm-core/src/jvm/backtype/storm/multilang/ShellProcess.java new file mode 100644 index 0000000000..12689d8163 --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/multilang/ShellProcess.java @@ -0,0 +1,101 @@ +package backtype.storm.multilang; + +import backtype.storm.task.TopologyContext; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.IOUtils; +import org.apache.log4j.Logger; + +public class ShellProcess implements Serializable { + public static Logger LOG = Logger.getLogger(ShellProcess.class); + private Process _subprocess; + private InputStream processErrorStream; + private String[] command; + public ISerializer serializer; + + public ShellProcess(ISerializer serializer, String[] command) { + this.command = command; + this.serializer = serializer; + } + + public Number launch(Map conf, TopologyContext context) { + ProcessBuilder builder = new ProcessBuilder(command); + builder.directory(new File(context.getCodeDir())); + + Number pid; + try { + _subprocess = builder.start(); + processErrorStream = _subprocess.getErrorStream(); + serializer.initialize(_subprocess.getOutputStream(), _subprocess.getInputStream()); + pid = serializer.connect(conf, context); + } catch (IOException e) { + throw new RuntimeException("Error when launching multilang subprocess\n" + getErrorsString(), e); + } catch (NoOutputException e) { + throw new RuntimeException(e + getErrorsString() + "\n"); + } + return pid; + } + + public void destroy() { + _subprocess.destroy(); + } + + public Emission readEmission() throws IOException { + try { + return serializer.readEmission(); + } catch (NoOutputException e) { + throw new RuntimeException(e + getErrorsString() + "\n"); + } + } + + public void writeImmission(Immission msg) throws IOException { + serializer.writeImmission(msg); + // drain the error stream to avoid dead lock because of full error stream buffer + drainErrorStream(); + } + + public void writeSpoutMsg(SpoutMsg msg) throws IOException { + serializer.writeSpoutMsg(msg); + // drain the error stream to avoid dead lock because of full error stream buffer + drainErrorStream(); + } + + public void writeTaskIds(List taskIds) throws IOException { + serializer.writeTaskIds(taskIds); + // drain the error stream to avoid dead lock because of full error stream buffer + drainErrorStream(); + } + + public void drainErrorStream() + { + try { + while (processErrorStream.available() > 0) + { + int bufferSize = processErrorStream.available(); + byte[] errorReadingBuffer = new byte[bufferSize]; + + processErrorStream.read(errorReadingBuffer, 0, bufferSize); + + LOG.info("Got error from shell process: " + new String(errorReadingBuffer)); + } + } catch(Exception e) { + } + } + + public String getErrorsString() { + if(processErrorStream!=null) { + try { + return IOUtils.toString(processErrorStream); + } catch(IOException e) { + return "(Unable to capture error stream)"; + } + } else { + return ""; + } + } +} diff --git a/storm-core/src/jvm/backtype/storm/multilang/ShellSpout.java b/storm-core/src/jvm/backtype/storm/multilang/ShellSpout.java new file mode 100644 index 0000000000..7895a004e9 --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/multilang/ShellSpout.java @@ -0,0 +1,115 @@ +package backtype.storm.multilang; + +import backtype.storm.generated.ShellComponent; +import backtype.storm.spout.ISpout; +import backtype.storm.spout.SpoutOutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.multilang.ShellProcess; +import backtype.storm.utils.Utils; +import java.util.Map; +import java.util.List; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class ShellSpout implements ISpout { + public static Logger LOG = LoggerFactory.getLogger(ShellSpout.class); + + private SpoutOutputCollector _collector; + private String[] _command; + private ShellProcess _process; + private SpoutMsg spoutMsg; + + public ShellSpout(ShellComponent component) { + this(component.get_execution_command(), component.get_script()); + _process = new ShellProcess(new JsonSerializer(), _command); + } + + public ShellSpout(String... command) { + _command = command; + _process = new ShellProcess(new JsonSerializer(), _command); + } + + public ShellSpout(ISerializer serializer, String... command) { + _command = command; + _process = new ShellProcess(serializer, _command); + } + + public void open(Map stormConf, TopologyContext context, + SpoutOutputCollector collector) { + _collector = collector; + + Number subpid = _process.launch(stormConf, context); + LOG.info("Launched subprocess with pid " + subpid); + } + + public void close() { + _process.destroy(); + } + + public void nextTuple() { + if (spoutMsg == null) { + spoutMsg = new SpoutMsg(); + } + spoutMsg.setCommand("next"); + spoutMsg.setId(""); + querySubprocess(); + } + + public void ack(Object msgId) { + if (spoutMsg == null) { + spoutMsg = new SpoutMsg(); + } + spoutMsg.setCommand("ack"); + spoutMsg.setId(msgId.toString()); + querySubprocess(); + } + + public void fail(Object msgId) { + if (spoutMsg == null) { + spoutMsg = new SpoutMsg(); + } + spoutMsg.setCommand("fail"); + spoutMsg.setId(msgId.toString()); + querySubprocess(); + } + + private void querySubprocess() { + try { + _process.writeSpoutMsg(spoutMsg); + + while (true) { + Emission emission = _process.readEmission(); + String command = emission.getCommand(); + if (command.equals("sync")) { + return; + } else if (command.equals("log")) { + String msg = emission.getMsg(); + LOG.info("Shell msg: " + msg); + } else if (command.equals("emit")) { + String stream = emission.getStream(); + Long task = emission.getTask(); + List tuple = emission.getTuple(); + Object messageId = emission.getId(); + if (task == 0) { + List outtasks = _collector.emit(stream, tuple, messageId); + _process.writeTaskIds(outtasks); + } else { + _collector.emitDirect((int)task.longValue(), stream, tuple, messageId); + } + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void activate() { + } + + @Override + public void deactivate() { + } +} diff --git a/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java b/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java new file mode 100644 index 0000000000..00967cb31c --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java @@ -0,0 +1,18 @@ +package backtype.storm.multilang; + +public class SpoutMsg { + private String command; + private String id; + public String getCommand() { + return command; + } + public void setCommand(String command) { + this.command = command; + } + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } +} diff --git a/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java b/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java deleted file mode 100644 index ba7bb63a8b..0000000000 --- a/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java +++ /dev/null @@ -1,120 +0,0 @@ -package backtype.storm.spout; - -import backtype.storm.generated.ShellComponent; -import backtype.storm.task.TopologyContext; -import backtype.storm.utils.ShellProcess; -import backtype.storm.utils.Utils; -import java.util.Map; -import java.util.List; -import java.io.IOException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.json.simple.JSONObject; - - -public class ShellSpout implements ISpout { - public static Logger LOG = LoggerFactory.getLogger(ShellSpout.class); - - private SpoutOutputCollector _collector; - private String[] _command; - private ShellProcess _process; - - public ShellSpout(ShellComponent component) { - this(component.get_execution_command(), component.get_script()); - } - - public ShellSpout(String... command) { - _command = command; - } - - public void open(Map stormConf, TopologyContext context, - SpoutOutputCollector collector) { - _process = new ShellProcess(_command); - _collector = collector; - - try { - Number subpid = _process.launch(stormConf, context); - LOG.info("Launched subprocess with pid " + subpid); - } catch (IOException e) { - throw new RuntimeException("Error when launching multilang subprocess\n" + _process.getErrorsString(), e); - } - } - - public void close() { - _process.destroy(); - } - - private JSONObject _next; - public void nextTuple() { - if (_next == null) { - _next = new JSONObject(); - _next.put("command", "next"); - } - - querySubprocess(_next); - } - - private JSONObject _ack; - public void ack(Object msgId) { - if (_ack == null) { - _ack = new JSONObject(); - _ack.put("command", "ack"); - } - - _ack.put("id", msgId); - querySubprocess(_ack); - } - - private JSONObject _fail; - public void fail(Object msgId) { - if (_fail == null) { - _fail = new JSONObject(); - _fail.put("command", "fail"); - } - - _fail.put("id", msgId); - querySubprocess(_fail); - } - - private void querySubprocess(Object query) { - try { - _process.writeMessage(query); - - while (true) { - JSONObject action = _process.readMessage(); - String command = (String) action.get("command"); - if (command.equals("sync")) { - return; - } else if (command.equals("log")) { - String msg = (String) action.get("msg"); - LOG.info("Shell msg: " + msg); - } else if (command.equals("emit")) { - String stream = (String) action.get("stream"); - if (stream == null) stream = Utils.DEFAULT_STREAM_ID; - Long task = (Long) action.get("task"); - List tuple = (List) action.get("tuple"); - Object messageId = (Object) action.get("id"); - if (task == null) { - List outtasks = _collector.emit(stream, tuple, messageId); - Object need_task_ids = action.get("need_task_ids"); - if (need_task_ids == null || ((Boolean) need_task_ids).booleanValue()) { - _process.writeMessage(outtasks); - } - } else { - _collector.emitDirect((int)task.longValue(), stream, tuple, messageId); - } - } - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public void activate() { - } - - @Override - public void deactivate() { - } -} diff --git a/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java b/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java deleted file mode 100644 index 49c428ae39..0000000000 --- a/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java +++ /dev/null @@ -1,132 +0,0 @@ -package backtype.storm.utils; - -import backtype.storm.task.TopologyContext; -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.DataOutputStream; -import java.io.File; -import java.io.IOException; -import java.util.Map; -import java.util.List; - -import org.apache.commons.io.IOUtils; -import org.json.simple.JSONObject; -import org.json.simple.JSONValue; -import org.apache.log4j.Logger; - -public class ShellProcess { - public static Logger LOG = Logger.getLogger(ShellProcess.class); - private DataOutputStream processIn; - private BufferedReader processOut; - private InputStream processErrorStream; - private Process _subprocess; - private String[] command; - - public ShellProcess(String[] command) { - this.command = command; - } - - public Number launch(Map conf, TopologyContext context) throws IOException { - ProcessBuilder builder = new ProcessBuilder(command); - builder.directory(new File(context.getCodeDir())); - _subprocess = builder.start(); - - processIn = new DataOutputStream(_subprocess.getOutputStream()); - processOut = new BufferedReader(new InputStreamReader(_subprocess.getInputStream())); - processErrorStream = _subprocess.getErrorStream(); - - JSONObject setupInfo = new JSONObject(); - setupInfo.put("pidDir", context.getPIDDir()); - setupInfo.put("conf", conf); - setupInfo.put("context", context); - writeMessage(setupInfo); - - return (Number)readMessage().get("pid"); - } - - public void destroy() { - _subprocess.destroy(); - } - - public void writeMessage(Object msg) throws IOException { - writeString(JSONValue.toJSONString(msg)); - } - - private void writeString(String str) throws IOException { - byte[] strBytes = str.getBytes("UTF-8"); - processIn.write(strBytes, 0, strBytes.length); - processIn.writeBytes("\nend\n"); - processIn.flush(); - } - - public JSONObject readMessage() throws IOException { - String string = readString(); - JSONObject msg = (JSONObject)JSONValue.parse(string); - if (msg != null) { - return msg; - } else { - throw new IOException("unable to parse: " + string); - } - } - - public String getErrorsString() { - if(processErrorStream!=null) { - try { - return IOUtils.toString(processErrorStream); - } catch(IOException e) { - return "(Unable to capture error stream)"; - } - } else { - return ""; - } - } - - public void drainErrorStream() - { - try { - while (processErrorStream.available() > 0) - { - int bufferSize = processErrorStream.available(); - byte[] errorReadingBuffer = new byte[bufferSize]; - - processErrorStream.read(errorReadingBuffer, 0, bufferSize); - - LOG.info("Got error from shell process: " + new String(errorReadingBuffer)); - } - } catch(Exception e) { - } - } - - private String readString() throws IOException { - StringBuilder line = new StringBuilder(); - - //synchronized (processOut) { - while (true) { - String subline = processOut.readLine(); - if(subline==null) { - StringBuilder errorMessage = new StringBuilder(); - errorMessage.append("Pipe to subprocess seems to be broken!"); - if (line.length() == 0) { - errorMessage.append(" No output read.\n"); - } - else { - errorMessage.append(" Currently read output: " + line.toString() + "\n"); - } - errorMessage.append("Shell Process Exception:\n"); - errorMessage.append(getErrorsString() + "\n"); - throw new RuntimeException(errorMessage.toString()); - } - if(subline.equals("end")) { - break; - } - if(line.length()!=0) { - line.append("\n"); - } - line.append(subline); - } - //} - - return line.toString(); - } -} From a7918f8ef91b82b5b8fd91c91b36378a6c99caad Mon Sep 17 00:00:00 2001 From: John Gilmore Date: Mon, 7 Oct 2013 09:33:53 +0200 Subject: [PATCH 220/349] Moved ShellBolt, ShellProcess and ShellSpout back to their old packages. Updated dependencies. --- storm-core/src/clj/backtype/storm/bootstrap.clj | 7 +++---- .../src/jvm/backtype/storm/clojure/RichShellBolt.java | 2 +- .../jvm/backtype/storm/clojure/RichShellSpout.java | 2 +- .../storm/{multilang => spout}/ShellSpout.java | 10 ++++++---- .../backtype/storm/{multilang => task}/ShellBolt.java | 11 ++++++----- .../storm/{multilang => utils}/ShellProcess.java | 7 ++++++- 6 files changed, 23 insertions(+), 16 deletions(-) rename storm-core/src/jvm/backtype/storm/{multilang => spout}/ShellSpout.java (93%) rename storm-core/src/jvm/backtype/storm/{multilang => task}/ShellBolt.java (96%) rename storm-core/src/jvm/backtype/storm/{multilang => utils}/ShellProcess.java (92%) diff --git a/storm-core/src/clj/backtype/storm/bootstrap.clj b/storm-core/src/clj/backtype/storm/bootstrap.clj index 1075c664df..9edbe8e54c 100644 --- a/storm-core/src/clj/backtype/storm/bootstrap.clj +++ b/storm-core/src/clj/backtype/storm/bootstrap.clj @@ -12,13 +12,12 @@ RegisteredGlobalState ThriftTopologyUtils DisruptorQueue MutableObject MutableLong])) (import (quote [backtype.storm.serialization KryoTupleSerializer KryoTupleDeserializer])) - (import (quote [backtype.storm.spout ISpout SpoutOutputCollector ISpoutOutputCollector])) - (import (quote [backtype.storm.multilang ShellSpout ShellBolt])) + (import (quote [backtype.storm.spout ISpout SpoutOutputCollector ISpoutOutputCollector ShellSpout])) (import (quote [backtype.storm.tuple Tuple TupleImpl Fields MessageId])) (import (quote [backtype.storm.task IBolt IOutputCollector - OutputCollector TopologyContext + OutputCollector TopologyContext ShellBolt GeneralTopologyContext WorkerTopologyContext])) - (import (quote [backtype.storm.coordination CoordinatedBolt CoordinatedBolt$SourceArgs + (import (quote [backtype.storm.coordination CoordinatedBolt CoordinatedBolt$SourceArgs IBatchBolt BatchBoltExecutor])) (import (quote [backtype.storm.drpc KeyedFairBolt])) (import (quote [backtype.storm.daemon Shutdownable])) diff --git a/storm-core/src/jvm/backtype/storm/clojure/RichShellBolt.java b/storm-core/src/jvm/backtype/storm/clojure/RichShellBolt.java index 6ff669eaf4..6be104ea1f 100644 --- a/storm-core/src/jvm/backtype/storm/clojure/RichShellBolt.java +++ b/storm-core/src/jvm/backtype/storm/clojure/RichShellBolt.java @@ -1,7 +1,7 @@ package backtype.storm.clojure; import backtype.storm.generated.StreamInfo; -import backtype.storm.multilang.ShellBolt; +import backtype.storm.task.ShellBolt; import backtype.storm.topology.IRichBolt; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.tuple.Fields; diff --git a/storm-core/src/jvm/backtype/storm/clojure/RichShellSpout.java b/storm-core/src/jvm/backtype/storm/clojure/RichShellSpout.java index 57410e1e9c..cb5947f692 100644 --- a/storm-core/src/jvm/backtype/storm/clojure/RichShellSpout.java +++ b/storm-core/src/jvm/backtype/storm/clojure/RichShellSpout.java @@ -1,7 +1,7 @@ package backtype.storm.clojure; import backtype.storm.generated.StreamInfo; -import backtype.storm.multilang.ShellSpout; +import backtype.storm.spout.ShellSpout; import backtype.storm.topology.IRichSpout; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.tuple.Fields; diff --git a/storm-core/src/jvm/backtype/storm/multilang/ShellSpout.java b/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java similarity index 93% rename from storm-core/src/jvm/backtype/storm/multilang/ShellSpout.java rename to storm-core/src/jvm/backtype/storm/spout/ShellSpout.java index 7895a004e9..c9ef682b3d 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/ShellSpout.java +++ b/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java @@ -1,10 +1,12 @@ -package backtype.storm.multilang; +package backtype.storm.spout; import backtype.storm.generated.ShellComponent; -import backtype.storm.spout.ISpout; -import backtype.storm.spout.SpoutOutputCollector; +import backtype.storm.multilang.Emission; +import backtype.storm.multilang.ISerializer; +import backtype.storm.multilang.JsonSerializer; +import backtype.storm.multilang.SpoutMsg; import backtype.storm.task.TopologyContext; -import backtype.storm.multilang.ShellProcess; +import backtype.storm.utils.ShellProcess; import backtype.storm.utils.Utils; import java.util.Map; import java.util.List; diff --git a/storm-core/src/jvm/backtype/storm/multilang/ShellBolt.java b/storm-core/src/jvm/backtype/storm/task/ShellBolt.java similarity index 96% rename from storm-core/src/jvm/backtype/storm/multilang/ShellBolt.java rename to storm-core/src/jvm/backtype/storm/task/ShellBolt.java index acd60c9809..36585e3316 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/ShellBolt.java +++ b/storm-core/src/jvm/backtype/storm/task/ShellBolt.java @@ -1,11 +1,12 @@ -package backtype.storm.multilang; +package backtype.storm.task; import backtype.storm.generated.ShellComponent; -import backtype.storm.task.IBolt; -import backtype.storm.task.OutputCollector; -import backtype.storm.task.TopologyContext; import backtype.storm.tuple.Tuple; -import backtype.storm.multilang.ShellProcess; +import backtype.storm.utils.ShellProcess; +import backtype.storm.multilang.Emission; +import backtype.storm.multilang.ISerializer; +import backtype.storm.multilang.Immission; +import backtype.storm.multilang.JsonSerializer; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ConcurrentHashMap; diff --git a/storm-core/src/jvm/backtype/storm/multilang/ShellProcess.java b/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java similarity index 92% rename from storm-core/src/jvm/backtype/storm/multilang/ShellProcess.java rename to storm-core/src/jvm/backtype/storm/utils/ShellProcess.java index 12689d8163..38976bcb59 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/ShellProcess.java +++ b/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java @@ -1,5 +1,10 @@ -package backtype.storm.multilang; +package backtype.storm.utils; +import backtype.storm.multilang.Emission; +import backtype.storm.multilang.ISerializer; +import backtype.storm.multilang.Immission; +import backtype.storm.multilang.NoOutputException; +import backtype.storm.multilang.SpoutMsg; import backtype.storm.task.TopologyContext; import java.io.File; import java.io.IOException; From 3701bfcbe3ed90e1ddf73605a0e5cd0bf4f0dbd4 Mon Sep 17 00:00:00 2001 From: John Gilmore Date: Mon, 7 Oct 2013 11:47:48 +0200 Subject: [PATCH 221/349] Added comments to the new multilang objects. --- project.clj | 2 +- .../src/jvm/backtype/storm/multilang/Emission.java | 12 +++++++++++- .../jvm/backtype/storm/multilang/ISerializer.java | 5 +++++ .../src/jvm/backtype/storm/multilang/Immission.java | 12 +++++++++++- .../jvm/backtype/storm/multilang/JsonSerializer.java | 3 +++ .../backtype/storm/multilang/NoOutputException.java | 4 ++++ .../src/jvm/backtype/storm/multilang/SpoutMsg.java | 10 ++++++++++ 7 files changed, 45 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index f913924b0c..6e59efca6e 100644 --- a/project.clj +++ b/project.clj @@ -10,7 +10,7 @@ :archive "https://groups.google.com/group/storm-user" :post "storm-user@googlegroups.com"} :dependencies [~@DEPENDENCIES] - :plugins [[~'lein-sub "0.2.1"],[no-man-is-an-island/lein-eclipse "2.0.0"]] + :plugins [[~'lein-sub "0.2.1"]] :min-lein-version "2.0.0" :sub [~@MODULES] )) diff --git a/storm-core/src/jvm/backtype/storm/multilang/Emission.java b/storm-core/src/jvm/backtype/storm/multilang/Emission.java index 10544580be..d477d8b6eb 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/Emission.java +++ b/storm-core/src/jvm/backtype/storm/multilang/Emission.java @@ -3,6 +3,16 @@ import java.util.ArrayList; import java.util.List; +/** + * Emission is an object that represents the data sent to a shell component + * from a process that implements a multi-language protocol. It is the union + * of all data types that a component can send to Storm. + * + *

Emissions are objects received from the ISerializer interface, after + * the serializer has deserialized the data from the underlying wire protocol. + * The Emission class allows for a decoupling between the serialized + * representation of the data and the data itself.

+ */ public class Emission { private String command; private String id; @@ -11,7 +21,7 @@ public class Emission { private long task; private String msg; private List tuple; - + public String getCommand() { return command; } diff --git a/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java b/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java index 61a28a930e..691b432e32 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java +++ b/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java @@ -9,6 +9,11 @@ import backtype.storm.task.TopologyContext; +/** + * The ISerializer interface describes the methods that an object should implement + * to provide serialization and de-serialization capabilities to non-JVM + * language components. + */ public interface ISerializer extends Serializable { void initialize (OutputStream processIn, InputStream processOut); Number connect (Map conf, TopologyContext context) throws IOException, NoOutputException; diff --git a/storm-core/src/jvm/backtype/storm/multilang/Immission.java b/storm-core/src/jvm/backtype/storm/multilang/Immission.java index c20df53506..968f7286c6 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/Immission.java +++ b/storm-core/src/jvm/backtype/storm/multilang/Immission.java @@ -2,13 +2,23 @@ import java.util.List; +/** + * Immission is an object that represents the data sent from a shell component + * to a process that implements a multi-language protocol. It is the union + * of all data types that a component can receive from Storm. + * + *

Immissions are objects sent to the ISerializer interface, for + * serialization according to the wire protocol implemented by the serializer. + * The Immission class allows for a decoupling between the serialized + * representation of the data and the data itself.

+ */ public class Immission { private String id; private String comp; private String stream; private long task; private List tuple; - + public String getId() { return id; } diff --git a/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java b/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java index 88eead29ad..ba574e577e 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java +++ b/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java @@ -18,6 +18,9 @@ import backtype.storm.tuple.Tuple; import backtype.storm.utils.Utils; +/** + * JsonSerializer implements the JSON multilang protocol. + */ public class JsonSerializer implements ISerializer { private DataOutputStream processIn; private BufferedReader processOut; diff --git a/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java b/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java index 595a0ae7e2..8ffb7dd432 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java +++ b/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java @@ -1,5 +1,9 @@ package backtype.storm.multilang; +/** + * A NoOutputException states that no data has been received from the connected + * non-JVM process. + */ public class NoOutputException extends Exception { public NoOutputException() { super(); } public NoOutputException(String message) { super(message); } diff --git a/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java b/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java index 00967cb31c..c8063ebf36 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java +++ b/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java @@ -1,5 +1,15 @@ package backtype.storm.multilang; +/** + * SpoutMsg is an object that represents the data sent from a shell spout + * to a process that implements a multi-language spout. The SpoutMsg is used + * to send a "next", "ack" or "fail" message to a spout. + * + *

Spout messages are objects sent to the ISerializer interface, for + * serialization according to the wire protocol implemented by the serializer. + * The SpoutMsg class allows for a decoupling between the serialized + * representation of the data and the data itself.

+ */ public class SpoutMsg { private String command; private String id; From 3219a4f05f938f4ce5bdbb9f006e49cbec5ee89f Mon Sep 17 00:00:00 2001 From: John Gilmore Date: Tue, 8 Oct 2013 13:52:00 +0200 Subject: [PATCH 222/349] multilang serializer is not handled by stormConf. Renamed Emission and Immission to ShellMsg and BoltMsg respectively. Documented ISerializer interface. --- conf/defaults.yaml | 1 + storm-core/src/jvm/backtype/storm/Config.java | 129 ++++++++-------- .../jvm/backtype/storm/multilang/BoltMsg.java | 63 ++++++++ .../backtype/storm/multilang/Emission.java | 79 ---------- .../backtype/storm/multilang/ISerializer.java | 59 ++++++-- .../backtype/storm/multilang/Immission.java | 52 ------- .../storm/multilang/JsonSerializer.java | 138 +++++++++--------- .../storm/multilang/NoOutputException.java | 19 ++- .../backtype/storm/multilang/ShellMsg.java | 96 ++++++++++++ .../backtype/storm/multilang/SpoutMsg.java | 44 +++--- .../jvm/backtype/storm/spout/ShellSpout.java | 30 ++-- .../jvm/backtype/storm/task/ShellBolt.java | 65 ++++----- .../backtype/storm/utils/ShellProcess.java | 95 +++++++----- 13 files changed, 489 insertions(+), 381 deletions(-) create mode 100644 storm-core/src/jvm/backtype/storm/multilang/BoltMsg.java delete mode 100644 storm-core/src/jvm/backtype/storm/multilang/Emission.java delete mode 100644 storm-core/src/jvm/backtype/storm/multilang/Immission.java create mode 100644 storm-core/src/jvm/backtype/storm/multilang/ShellMsg.java diff --git a/conf/defaults.yaml b/conf/defaults.yaml index b17e445a2b..92e65d6d19 100644 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@ -19,6 +19,7 @@ storm.cluster.mode: "distributed" # can be distributed or local storm.local.mode.zmq: false storm.thrift.transport: "backtype.storm.security.auth.SimpleTransportPlugin" storm.messaging.transport: "backtype.storm.messaging.zmq" +storm.multilang.serializer: "backtype.storm.multilang.JsonSerializer" ### nimbus.* configs are for the master nimbus.host: "localhost" diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java index 564ad0d330..fff157363e 100644 --- a/storm-core/src/jvm/backtype/storm/Config.java +++ b/storm-core/src/jvm/backtype/storm/Config.java @@ -10,11 +10,11 @@ import java.util.Map; /** - * Topology configs are specified as a plain old map. This class provides a - * convenient way to create a topology config map by providing setter methods for - * all the configs that can be set. It also makes it easier to do things like add + * Topology configs are specified as a plain old map. This class provides a + * convenient way to create a topology config map by providing setter methods for + * all the configs that can be set. It also makes it easier to do things like add * serializations. - * + * *

This class also provides constants for all the configurations possible on * a Storm cluster and Storm topology. Each constant is paired with a schema * that defines the validity criterion of the corresponding field. Default @@ -22,10 +22,17 @@ * *

Note that you may put other configurations in any of the configs. Storm * will ignore anything it doesn't recognize, but your topologies are free to make - * use of them by reading them in the prepare method of Bolts or the open method of + * use of them by reading them in the prepare method of Bolts or the open method of * Spouts.

*/ public class Config extends HashMap { + /** + * The serializer for communication between shell components and non-JVM + * processes + */ + public static final String STORM_MULTILANG_SERIALIZER = "storm.multilang.serializer"; + public static final Object STORM_MULTILANG_SERIALIZER_SCHEMA = String.class; + /** * The transporter for communication among Storm tasks */ @@ -35,25 +42,25 @@ public class Config extends HashMap { /** * Netty based messaging: The buffer size for send/recv buffer */ - public static final String STORM_MESSAGING_NETTY_BUFFER_SIZE = "storm.messaging.netty.buffer_size"; + public static final String STORM_MESSAGING_NETTY_BUFFER_SIZE = "storm.messaging.netty.buffer_size"; public static final Object STORM_MESSAGING_NETTY_BUFFER_SIZE_SCHEMA = Number.class; /** * Netty based messaging: The max # of retries that a peer will perform when a remote is not accessible */ - public static final String STORM_MESSAGING_NETTY_MAX_RETRIES = "storm.messaging.netty.max_retries"; + public static final String STORM_MESSAGING_NETTY_MAX_RETRIES = "storm.messaging.netty.max_retries"; public static final Object STORM_MESSAGING_NETTY_MAX_RETRIES_SCHEMA = Number.class; /** - * Netty based messaging: The min # of milliseconds that a peer will wait. + * Netty based messaging: The min # of milliseconds that a peer will wait. */ - public static final String STORM_MESSAGING_NETTY_MIN_SLEEP_MS = "storm.messaging.netty.min_wait_ms"; + public static final String STORM_MESSAGING_NETTY_MIN_SLEEP_MS = "storm.messaging.netty.min_wait_ms"; public static final Object STORM_MESSAGING_NETTY_MIN_SLEEP_MS_SCHEMA = Number.class; /** - * Netty based messaging: The max # of milliseconds that a peer will wait. + * Netty based messaging: The max # of milliseconds that a peer will wait. */ - public static final String STORM_MESSAGING_NETTY_MAX_SLEEP_MS = "storm.messaging.netty.max_wait_ms"; + public static final String STORM_MESSAGING_NETTY_MAX_SLEEP_MS = "storm.messaging.netty.max_wait_ms"; public static final Object STORM_MESSAGING_NETTY_MAX_SLEEP_MS_SCHEMA = Number.class; /** @@ -78,7 +85,7 @@ public class Config extends HashMap { /** * A global task scheduler used to assign topologies's tasks to supervisors' wokers. - * + * * If this is not set, a default system scheduler will be used. */ public static final String STORM_SCHEDULER = "storm.scheduler"; @@ -91,9 +98,9 @@ public class Config extends HashMap { public static final Object STORM_CLUSTER_MODE_SCHEMA = String.class; /** - * The hostname the supervisors/workers should report to nimbus. If unset, Storm will + * The hostname the supervisors/workers should report to nimbus. If unset, Storm will * get the hostname to report by calling InetAddress.getLocalHost().getCanonicalHostName(). - * + * * You should set this config when you dont have a DNS which supervisors/workers * can utilize to find each other based on hostname got from calls to * InetAddress.getLocalHost().getCanonicalHostName(). @@ -108,16 +115,16 @@ public class Config extends HashMap { public static final Object STORM_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class; /** - * The serializer class for ListDelegate (tuple payload). + * The serializer class for ListDelegate (tuple payload). * The default serializer will be ListDelegateSerializer */ public static final String TOPOLOGY_TUPLE_SERIALIZER = "topology.tuple.serializer"; public static final Object TOPOLOGY_TUPLE_SERIALIZER_SCHEMA = String.class; /** - * Whether or not to use ZeroMQ for messaging in local mode. If this is set - * to false, then Storm will use a pure-Java messaging system. The purpose - * of this flag is to make it easy to run Storm in local mode by eliminating + * Whether or not to use ZeroMQ for messaging in local mode. If this is set + * to false, then Storm will use a pure-Java messaging system. The purpose + * of this flag is to make it easy to run Storm in local mode by eliminating * the need for native dependencies, which can be difficult to install. * * Defaults to false. @@ -255,7 +262,7 @@ public class Config extends HashMap { public static final Object NIMBUS_TASK_LAUNCH_SECS_SCHEMA = Number.class; /** - * Whether or not nimbus should reassign tasks if it detects that a task goes down. + * Whether or not nimbus should reassign tasks if it detects that a task goes down. * Defaults to true, and it's not recommended to change this value. */ public static final String NIMBUS_REASSIGN = "nimbus.reassign"; @@ -319,19 +326,19 @@ public class Config extends HashMap { public static final Object DRPC_PORT_SCHEMA = Number.class; /** - * DRPC thrift server worker threads + * DRPC thrift server worker threads */ public static final String DRPC_WORKER_THREADS = "drpc.worker.threads"; public static final Object DRPC_WORKER_THREADS_SCHEMA = Number.class; /** - * DRPC thrift server queue size + * DRPC thrift server queue size */ public static final String DRPC_QUEUE_SIZE = "drpc.queue.size"; public static final Object DRPC_QUEUE_SIZE_SCHEMA = Number.class; /** - * This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back. + * This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back. */ public static final String DRPC_INVOCATIONS_PORT = "drpc.invocations.port"; public static final Object DRPC_INVOCATIONS_PORT_SCHEMA = Number.class; @@ -479,8 +486,8 @@ public class Config extends HashMap { /** * How many instances to create for a spout/bolt. A task runs on a thread with zero or more * other tasks for the same spout/bolt. The number of tasks for a spout/bolt is always - * the same throughout the lifetime of a topology, but the number of executors (threads) for - * a spout/bolt can change over time. This allows a topology to scale to more or less resources + * the same throughout the lifetime of a topology, but the number of executors (threads) for + * a spout/bolt can change over time. This allows a topology to scale to more or less resources * without redeploying the topology or violating the constraints of Storm (such as a fields grouping * guaranteeing that the same value goes to the same task). */ @@ -519,8 +526,8 @@ public class Config extends HashMap { /** * A list of classes that customize storm's kryo instance during start-up. - * Each listed class name must implement IKryoDecorator. During start-up the - * listed class is instantiated with 0 arguments, then its 'decorate' method + * Each listed class name must implement IKryoDecorator. During start-up the + * listed class is instantiated with 0 arguments, then its 'decorate' method * is called with storm's kryo instance as the only argument. */ public static final String TOPOLOGY_KRYO_DECORATORS = "topology.kryo.decorators"; @@ -550,7 +557,7 @@ public class Config extends HashMap { /* * A list of classes implementing IMetricsConsumer (See storm.yaml.example for exact config format). - * Each listed class will be routed all the metrics data generated by the storm metrics API. + * Each listed class will be routed all the metrics data generated by the storm metrics API. * Each listed class maps 1:1 to a system bolt named __metrics_ClassName#N, and it's parallelism is configurable. */ public static final String TOPOLOGY_METRICS_CONSUMER_REGISTER = "topology.metrics.consumer.register"; @@ -566,24 +573,24 @@ public class Config extends HashMap { /** - * The maximum number of tuples that can be pending on a spout task at any given time. - * This config applies to individual tasks, not to spouts or topologies as a whole. - * + * The maximum number of tuples that can be pending on a spout task at any given time. + * This config applies to individual tasks, not to spouts or topologies as a whole. + * * A pending tuple is one that has been emitted from a spout but has not been acked or failed yet. - * Note that this config parameter has no effect for unreliable spouts that don't tag + * Note that this config parameter has no effect for unreliable spouts that don't tag * their tuples with a message id. */ - public static final String TOPOLOGY_MAX_SPOUT_PENDING="topology.max.spout.pending"; + public static final String TOPOLOGY_MAX_SPOUT_PENDING="topology.max.spout.pending"; public static final Object TOPOLOGY_MAX_SPOUT_PENDING_SCHEMA = Number.class; /** * A class that implements a strategy for what to do when a spout needs to wait. Waiting is * triggered in one of two conditions: - * + * * 1. nextTuple emits no tuples * 2. The spout has hit maxSpoutPending and can't emit any more tuples */ - public static final String TOPOLOGY_SPOUT_WAIT_STRATEGY="topology.spout.wait.strategy"; + public static final String TOPOLOGY_SPOUT_WAIT_STRATEGY="topology.spout.wait.strategy"; public static final Object TOPOLOGY_SPOUT_WAIT_STRATEGY_SCHEMA = String.class; /** @@ -606,7 +613,7 @@ public class Config extends HashMap { public static final Object TOPOLOGY_STATS_SAMPLE_RATE_SCHEMA = Number.class; /** - * The time period that builtin metrics data in bucketed into. + * The time period that builtin metrics data in bucketed into. */ public static final String TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS="topology.builtin.metrics.bucket.size.secs"; public static final Object TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS_SCHEMA = Number.class; @@ -633,7 +640,7 @@ public class Config extends HashMap { /** * A list of task hooks that are automatically added to every spout and bolt in the topology. An example - * of when you'd do this is to add a hook that integrates with your internal + * of when you'd do this is to add a hook that integrates with your internal * monitoring system. These hooks are instantiated using the zero-arg constructor. */ public static final String TOPOLOGY_AUTO_TASK_HOOKS="topology.auto.task.hooks"; @@ -647,7 +654,7 @@ public class Config extends HashMap { public static final Object TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE_SCHEMA = ConfigValidation.PowerOf2Validator; /** - * The maximum number of messages to batch from the thread receiving off the network to the + * The maximum number of messages to batch from the thread receiving off the network to the * executor queues. Must be a power of 2. */ public static final String TOPOLOGY_RECEIVER_BUFFER_SIZE="topology.receiver.buffer.size"; @@ -681,14 +688,14 @@ public class Config extends HashMap { public static final Object TOPOLOGY_DISRUPTOR_WAIT_STRATEGY_SCHEMA = String.class; /** - * The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed + * The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed * via the TopologyContext. */ public static final String TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE="topology.worker.shared.thread.pool.size"; public static final Object TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE_SCHEMA = Number.class; /** - * The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example, + * The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example, * an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be * reported to Zookeeper per task for every 10 second interval of time. */ @@ -757,9 +764,9 @@ public class Config extends HashMap { /** * This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers) - * for the java.library.path value. java.library.path tells the JVM where + * for the java.library.path value. java.library.path tells the JVM where * to look for native libraries. It is necessary to set this config correctly since - * Storm uses the ZeroMQ and JZMQ native libs. + * Storm uses the ZeroMQ and JZMQ native libs. */ public static final String JAVA_LIBRARY_PATH = "java.library.path"; public static final Object JAVA_LIBRARY_PATH_SCHEMA = String.class; @@ -781,17 +788,17 @@ public class Config extends HashMap { public static void setDebug(Map conf, boolean isOn) { conf.put(Config.TOPOLOGY_DEBUG, isOn); - } + } public void setDebug(boolean isOn) { setDebug(this, isOn); } - + @Deprecated public void setOptimize(boolean isOn) { put(Config.TOPOLOGY_OPTIMIZE, isOn); - } - + } + public static void setNumWorkers(Map conf, int workers) { conf.put(Config.TOPOLOGY_WORKERS, workers); } @@ -807,7 +814,7 @@ public static void setNumAckers(Map conf, int numExecutors) { public void setNumAckers(int numExecutors) { setNumAckers(this, numExecutors); } - + public static void setMessageTimeoutSecs(Map conf, int secs) { conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, secs); } @@ -815,7 +822,7 @@ public static void setMessageTimeoutSecs(Map conf, int secs) { public void setMessageTimeoutSecs(int secs) { setMessageTimeoutSecs(this, secs); } - + public static void registerSerialization(Map conf, Class klass) { getRegisteredSerializations(conf).add(klass.getName()); } @@ -823,17 +830,17 @@ public static void registerSerialization(Map conf, Class klass) { public void registerSerialization(Class klass) { registerSerialization(this, klass); } - + public static void registerSerialization(Map conf, Class klass, Class serializerClass) { Map register = new HashMap(); register.put(klass.getName(), serializerClass.getName()); - getRegisteredSerializations(conf).add(register); + getRegisteredSerializations(conf).add(register); } public void registerSerialization(Class klass, Class serializerClass) { registerSerialization(this, klass, serializerClass); } - + public void registerMetricsConsumer(Class klass, Object argument, long parallelismHint) { HashMap m = new HashMap(); m.put("class", klass.getCanonicalName()); @@ -861,7 +868,7 @@ public static void registerDecorator(Map conf, Class k public void registerDecorator(Class klass) { registerDecorator(this, klass); } - + public static void setKryoFactory(Map conf, Class klass) { conf.put(Config.TOPOLOGY_KRYO_FACTORY, klass.getName()); } @@ -877,7 +884,7 @@ public static void setSkipMissingKryoRegistrations(Map conf, boolean skip) { public void setSkipMissingKryoRegistrations(boolean skip) { setSkipMissingKryoRegistrations(this, skip); } - + public static void setMaxTaskParallelism(Map conf, int max) { conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, max); } @@ -885,7 +892,7 @@ public static void setMaxTaskParallelism(Map conf, int max) { public void setMaxTaskParallelism(int max) { setMaxTaskParallelism(this, max); } - + public static void setMaxSpoutPending(Map conf, int max) { conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, max); } @@ -893,23 +900,23 @@ public static void setMaxSpoutPending(Map conf, int max) { public void setMaxSpoutPending(int max) { setMaxSpoutPending(this, max); } - + public static void setStatsSampleRate(Map conf, double rate) { conf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, rate); - } + } public void setStatsSampleRate(double rate) { setStatsSampleRate(this, rate); - } + } public static void setFallBackOnJavaSerialization(Map conf, boolean fallback) { conf.put(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION, fallback); - } + } public void setFallBackOnJavaSerialization(boolean fallback) { setFallBackOnJavaSerialization(this, fallback); - } - + } + private static List getRegisteredSerializations(Map conf) { List ret; if(!conf.containsKey(Config.TOPOLOGY_KRYO_REGISTER)) { @@ -920,13 +927,13 @@ private static List getRegisteredSerializations(Map conf) { conf.put(Config.TOPOLOGY_KRYO_REGISTER, ret); return ret; } - + private static List getRegisteredDecorators(Map conf) { List ret; if(!conf.containsKey(Config.TOPOLOGY_KRYO_DECORATORS)) { ret = new ArrayList(); } else { - ret = new ArrayList((List) conf.get(Config.TOPOLOGY_KRYO_DECORATORS)); + ret = new ArrayList((List) conf.get(Config.TOPOLOGY_KRYO_DECORATORS)); } conf.put(Config.TOPOLOGY_KRYO_DECORATORS, ret); return ret; diff --git a/storm-core/src/jvm/backtype/storm/multilang/BoltMsg.java b/storm-core/src/jvm/backtype/storm/multilang/BoltMsg.java new file mode 100644 index 0000000000..1d6bd1d9e7 --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/multilang/BoltMsg.java @@ -0,0 +1,63 @@ +package backtype.storm.multilang; + +import java.util.List; + +/** + * BoltMsg is an object that represents the data sent from a shell component to + * a bolt process that implements a multi-language protocol. It is the union of + * all data types that a bolt can receive from Storm. + * + *

+ * BoltMsgs are objects sent to the ISerializer interface, for serialization + * according to the wire protocol implemented by the serializer. The BoltMsg + * class allows for a decoupling between the serialized representation of the + * data and the data itself. + *

+ */ +public class BoltMsg { + private String id; + private String comp; + private String stream; + private long task; + private List tuple; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getComp() { + return comp; + } + + public void setComp(String comp) { + this.comp = comp; + } + + public String getStream() { + return stream; + } + + public void setStream(String stream) { + this.stream = stream; + } + + public long getTask() { + return task; + } + + public void setTask(long task) { + this.task = task; + } + + public List getTuple() { + return tuple; + } + + public void setTuple(List tuple) { + this.tuple = tuple; + } +} diff --git a/storm-core/src/jvm/backtype/storm/multilang/Emission.java b/storm-core/src/jvm/backtype/storm/multilang/Emission.java deleted file mode 100644 index d477d8b6eb..0000000000 --- a/storm-core/src/jvm/backtype/storm/multilang/Emission.java +++ /dev/null @@ -1,79 +0,0 @@ -package backtype.storm.multilang; - -import java.util.ArrayList; -import java.util.List; - -/** - * Emission is an object that represents the data sent to a shell component - * from a process that implements a multi-language protocol. It is the union - * of all data types that a component can send to Storm. - * - *

Emissions are objects received from the ISerializer interface, after - * the serializer has deserialized the data from the underlying wire protocol. - * The Emission class allows for a decoupling between the serialized - * representation of the data and the data itself.

- */ -public class Emission { - private String command; - private String id; - private List anchors; - private String stream; - private long task; - private String msg; - private List tuple; - - public String getCommand() { - return command; - } - public void setCommand(String command) { - this.command = command; - } - public String getId() { - return id; - } - public void setId(String id) { - this.id = id; - } - public List getAnchors() { - return anchors; - } - public void setAnchors(List anchors) { - this.anchors = anchors; - } - public void addAnchor(String anchor) { - if (anchors == null) { - anchors = new ArrayList(); - } - this.anchors.add(anchor); - } - public String getStream() { - return stream; - } - public void setStream(String stream) { - this.stream = stream; - } - public long getTask() { - return task; - } - public void setTask(long task) { - this.task = task; - } - public String getMsg() { - return msg; - } - public void setMsg(String msg) { - this.msg = msg; - } - public List getTuple() { - return tuple; - } - public void setTuple(List tuple) { - this.tuple = tuple; - } - public void addTuple(Object tuple) { - if (this.tuple == null) { - this.tuple = new ArrayList(); - } - this.tuple.add(tuple); - } -} diff --git a/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java b/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java index 691b432e32..a6ccfd4326 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java +++ b/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java @@ -10,15 +10,56 @@ import backtype.storm.task.TopologyContext; /** - * The ISerializer interface describes the methods that an object should implement - * to provide serialization and de-serialization capabilities to non-JVM - * language components. + * The ISerializer interface describes the methods that an object should + * implement to provide serialization and de-serialization capabilities to + * non-JVM language components. */ public interface ISerializer extends Serializable { - void initialize (OutputStream processIn, InputStream processOut); - Number connect (Map conf, TopologyContext context) throws IOException, NoOutputException; - Emission readEmission() throws IOException, NoOutputException; - void writeImmission(Immission immission) throws IOException; - void writeSpoutMsg(SpoutMsg msg) throws IOException; - void writeTaskIds(List taskIds) throws IOException; + + /** + * This method sets the input and output streams of the serializer + * + * @param processIn output stream to non-JVM component + * @param processOut input stream from non-JVM component + */ + void initialize(OutputStream processIn, InputStream processOut); + + /** + * This method transmits the Storm config to the non-JVM process and + * receives its pid. + * + * @param conf storm configuration + * @param context topology context + * @return process pid + */ + Number connect(Map conf, TopologyContext context) throws IOException, + NoOutputException; + + /** + * This method receives a shell message from the non-JVM process + * + * @return shell message + */ + ShellMsg readShellMsg() throws IOException, NoOutputException; + + /** + * This method sends a bolt message to a non-JVM bolt process + * + * @param msg bolt message + */ + void writeBoltMsg(BoltMsg msg) throws IOException; + + /** + * This method sends a spout message to a non-JVM spout process + * + * @param msg spout message + */ + void writeSpoutMsg(SpoutMsg msg) throws IOException; + + /** + * This method sends a list of task IDs to a non-JVM bolt process + * + * @param taskIds list of task IDs + */ + void writeTaskIds(List taskIds) throws IOException; } diff --git a/storm-core/src/jvm/backtype/storm/multilang/Immission.java b/storm-core/src/jvm/backtype/storm/multilang/Immission.java deleted file mode 100644 index 968f7286c6..0000000000 --- a/storm-core/src/jvm/backtype/storm/multilang/Immission.java +++ /dev/null @@ -1,52 +0,0 @@ -package backtype.storm.multilang; - -import java.util.List; - -/** - * Immission is an object that represents the data sent from a shell component - * to a process that implements a multi-language protocol. It is the union - * of all data types that a component can receive from Storm. - * - *

Immissions are objects sent to the ISerializer interface, for - * serialization according to the wire protocol implemented by the serializer. - * The Immission class allows for a decoupling between the serialized - * representation of the data and the data itself.

- */ -public class Immission { - private String id; - private String comp; - private String stream; - private long task; - private List tuple; - - public String getId() { - return id; - } - public void setId(String id) { - this.id = id; - } - public String getComp() { - return comp; - } - public void setComp(String comp) { - this.comp = comp; - } - public String getStream() { - return stream; - } - public void setStream(String stream) { - this.stream = stream; - } - public long getTask() { - return task; - } - public void setTask(long task) { - this.task = task; - } - public List getTuple() { - return tuple; - } - public void setTuple(List tuple) { - this.tuple = tuple; - } -} diff --git a/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java b/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java index ba574e577e..ceb8cb3ed4 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java +++ b/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java @@ -22,126 +22,128 @@ * JsonSerializer implements the JSON multilang protocol. */ public class JsonSerializer implements ISerializer { - private DataOutputStream processIn; + private DataOutputStream processIn; private BufferedReader processOut; - public void initialize (OutputStream processIn, InputStream processOut) { - this.processIn = new DataOutputStream(processIn);; - this.processOut = new BufferedReader(new InputStreamReader(processOut)); - } + public void initialize(OutputStream processIn, InputStream processOut) { + this.processIn = new DataOutputStream(processIn); + this.processOut = new BufferedReader(new InputStreamReader(processOut)); + } - public Number connect (Map conf, TopologyContext context) throws IOException, NoOutputException { - JSONObject setupInfo = new JSONObject(); + public Number connect(Map conf, TopologyContext context) + throws IOException, NoOutputException { + JSONObject setupInfo = new JSONObject(); setupInfo.put("pidDir", context.getPIDDir()); setupInfo.put("conf", conf); setupInfo.put("context", context); - writeMessage(setupInfo); + writeMessage(setupInfo); - Number pid = (Number)((JSONObject)readMessage()).get("pid"); + Number pid = (Number) ((JSONObject) readMessage()).get("pid"); return pid; - } - - public void writeImmission(Immission immission) throws IOException { - JSONObject obj = new JSONObject(); - obj.put("id", immission.getId()); - obj.put("comp", immission.getComp()); - obj.put("stream", immission.getStream()); - obj.put("task", immission.getTask()); - obj.put("tuple", immission.getTuple()); + } + + public void writeBoltMsg(BoltMsg boltMsg) throws IOException { + JSONObject obj = new JSONObject(); + obj.put("id", boltMsg.getId()); + obj.put("comp", boltMsg.getComp()); + obj.put("stream", boltMsg.getStream()); + obj.put("task", boltMsg.getTask()); + obj.put("tuple", boltMsg.getTuple()); writeMessage(obj); - } + } - public void writeSpoutMsg(SpoutMsg msg) throws IOException { - JSONObject obj = new JSONObject(); + public void writeSpoutMsg(SpoutMsg msg) throws IOException { + JSONObject obj = new JSONObject(); obj.put("command", msg.getCommand()); obj.put("id", msg.getId()); writeMessage(obj); - } + } - public void writeTaskIds(List taskIds) throws IOException { + public void writeTaskIds(List taskIds) throws IOException { writeMessage(taskIds); - } + } - private void writeMessage(Object msg) throws IOException { + private void writeMessage(Object msg) throws IOException { writeString(JSONValue.toJSONString(msg)); } - private void writeString(String str) throws IOException { + private void writeString(String str) throws IOException { byte[] strBytes = str.getBytes("UTF-8"); processIn.write(strBytes, 0, strBytes.length); processIn.writeBytes("\nend\n"); processIn.flush(); } - public Emission readEmission() throws IOException, NoOutputException { - JSONObject msg = (JSONObject)readMessage(); - Emission emission = new Emission(); + public ShellMsg readShellMsg() throws IOException, NoOutputException { + JSONObject msg = (JSONObject) readMessage(); + ShellMsg shellMsg = new ShellMsg(); - String command = (String) msg.get("command"); - emission.setCommand(command); + String command = (String) msg.get("command"); + shellMsg.setCommand(command); - String stream = (String) msg.get("stream"); - if(stream == null) stream = Utils.DEFAULT_STREAM_ID; - emission.setStream(stream); + String stream = (String) msg.get("stream"); + if (stream == null) + stream = Utils.DEFAULT_STREAM_ID; + shellMsg.setStream(stream); Object taskObj = msg.get("task"); if (taskObj != null) { - emission.setTask((Long) taskObj); + shellMsg.setTask((Long) taskObj); } - emission.setTuple((List) msg.get("tuple")); + shellMsg.setTuple((List) msg.get("tuple")); List anchors = new ArrayList(); Object anchorObj = msg.get("anchors"); - if(anchorObj!=null) { - if(anchorObj instanceof String) { + if (anchorObj != null) { + if (anchorObj instanceof String) { anchorObj = Arrays.asList(anchorObj); } - for(Object o: (List) anchorObj) { - emission.addAnchor((String)o); + for (Object o : (List) anchorObj) { + shellMsg.addAnchor((String) o); } } - return emission; + return shellMsg; } - private Object readMessage() throws IOException, NoOutputException { - String string = readString(); + private Object readMessage() throws IOException, NoOutputException { + String string = readString(); Object msg = JSONValue.parse(string); if (msg != null) { - return msg; + return msg; } else { - throw new IOException("unable to parse: " + string); + throw new IOException("unable to parse: " + string); } - } + } private String readString() throws IOException, NoOutputException { StringBuilder line = new StringBuilder(); - //synchronized (processOut) { - while (true) { - String subline = processOut.readLine(); - if(subline==null) { - StringBuilder errorMessage = new StringBuilder(); - errorMessage.append("Pipe to subprocess seems to be broken!"); - if (line.length() == 0) { - errorMessage.append(" No output read.\n"); - } - else { - errorMessage.append(" Currently read output: " + line.toString() + "\n"); - } - errorMessage.append("Serializer Exception:\n"); - throw new NoOutputException(errorMessage.toString()); - } - if(subline.equals("end")) { - break; - } - if(line.length()!=0) { - line.append("\n"); + // synchronized (processOut) { + while (true) { + String subline = processOut.readLine(); + if (subline == null) { + StringBuilder errorMessage = new StringBuilder(); + errorMessage.append("Pipe to subprocess seems to be broken!"); + if (line.length() == 0) { + errorMessage.append(" No output read.\n"); + } else { + errorMessage.append(" Currently read output: " + + line.toString() + "\n"); } - line.append(subline); + errorMessage.append("Serializer Exception:\n"); + throw new NoOutputException(errorMessage.toString()); } - //} + if (subline.equals("end")) { + break; + } + if (line.length() != 0) { + line.append("\n"); + } + line.append(subline); + } + // } return line.toString(); } } diff --git a/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java b/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java index 8ffb7dd432..fd5ef72639 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java +++ b/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java @@ -5,8 +5,19 @@ * non-JVM process. */ public class NoOutputException extends Exception { - public NoOutputException() { super(); } - public NoOutputException(String message) { super(message); } - public NoOutputException(String message, Throwable cause) { super(message, cause); } - public NoOutputException(Throwable cause) { super(cause); } + public NoOutputException() { + super(); + } + + public NoOutputException(String message) { + super(message); + } + + public NoOutputException(String message, Throwable cause) { + super(message, cause); + } + + public NoOutputException(Throwable cause) { + super(cause); + } } \ No newline at end of file diff --git a/storm-core/src/jvm/backtype/storm/multilang/ShellMsg.java b/storm-core/src/jvm/backtype/storm/multilang/ShellMsg.java new file mode 100644 index 0000000000..d56ba7854c --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/multilang/ShellMsg.java @@ -0,0 +1,96 @@ +package backtype.storm.multilang; + +import java.util.ArrayList; +import java.util.List; + +/** + * ShellMsg is an object that represents the data sent to a shell component from + * a process that implements a multi-language protocol. It is the union of all + * data types that a component can send to Storm. + * + *

+ * ShellMsgs are objects received from the ISerializer interface, after the + * serializer has deserialized the data from the underlying wire protocol. The + * ShellMsg class allows for a decoupling between the serialized representation + * of the data and the data itself. + *

+ */ +public class ShellMsg { + private String command; + private String id; + private List anchors; + private String stream; + private long task; + private String msg; + private List tuple; + + public String getCommand() { + return command; + } + + public void setCommand(String command) { + this.command = command; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public List getAnchors() { + return anchors; + } + + public void setAnchors(List anchors) { + this.anchors = anchors; + } + + public void addAnchor(String anchor) { + if (anchors == null) { + anchors = new ArrayList(); + } + this.anchors.add(anchor); + } + + public String getStream() { + return stream; + } + + public void setStream(String stream) { + this.stream = stream; + } + + public long getTask() { + return task; + } + + public void setTask(long task) { + this.task = task; + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } + + public List getTuple() { + return tuple; + } + + public void setTuple(List tuple) { + this.tuple = tuple; + } + + public void addTuple(Object tuple) { + if (this.tuple == null) { + this.tuple = new ArrayList(); + } + this.tuple.add(tuple); + } +} diff --git a/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java b/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java index c8063ebf36..9b5ffce91e 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java +++ b/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java @@ -1,28 +1,34 @@ package backtype.storm.multilang; /** - * SpoutMsg is an object that represents the data sent from a shell spout - * to a process that implements a multi-language spout. The SpoutMsg is used - * to send a "next", "ack" or "fail" message to a spout. + * SpoutMsg is an object that represents the data sent from a shell spout to a + * process that implements a multi-language spout. The SpoutMsg is used to send + * a "next", "ack" or "fail" message to a spout. * - *

Spout messages are objects sent to the ISerializer interface, for + *

+ * Spout messages are objects sent to the ISerializer interface, for * serialization according to the wire protocol implemented by the serializer. * The SpoutMsg class allows for a decoupling between the serialized - * representation of the data and the data itself.

+ * representation of the data and the data itself. + *

*/ public class SpoutMsg { - private String command; - private String id; - public String getCommand() { - return command; - } - public void setCommand(String command) { - this.command = command; - } - public String getId() { - return id; - } - public void setId(String id) { - this.id = id; - } + private String command; + private String id; + + public String getCommand() { + return command; + } + + public void setCommand(String command) { + this.command = command; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } } diff --git a/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java b/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java index c9ef682b3d..72cebe28c2 100644 --- a/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java +++ b/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java @@ -1,13 +1,10 @@ package backtype.storm.spout; import backtype.storm.generated.ShellComponent; -import backtype.storm.multilang.Emission; -import backtype.storm.multilang.ISerializer; -import backtype.storm.multilang.JsonSerializer; +import backtype.storm.multilang.ShellMsg; import backtype.storm.multilang.SpoutMsg; import backtype.storm.task.TopologyContext; import backtype.storm.utils.ShellProcess; -import backtype.storm.utils.Utils; import java.util.Map; import java.util.List; import java.io.IOException; @@ -25,22 +22,16 @@ public class ShellSpout implements ISpout { public ShellSpout(ShellComponent component) { this(component.get_execution_command(), component.get_script()); - _process = new ShellProcess(new JsonSerializer(), _command); } public ShellSpout(String... command) { _command = command; - _process = new ShellProcess(new JsonSerializer(), _command); - } - - public ShellSpout(ISerializer serializer, String... command) { - _command = command; - _process = new ShellProcess(serializer, _command); } public void open(Map stormConf, TopologyContext context, SpoutOutputCollector collector) { _collector = collector; + _process = new ShellProcess(_command); Number subpid = _process.launch(stormConf, context); LOG.info("Launched subprocess with pid " + subpid); @@ -82,23 +73,24 @@ private void querySubprocess() { _process.writeSpoutMsg(spoutMsg); while (true) { - Emission emission = _process.readEmission(); - String command = emission.getCommand(); + ShellMsg shellMsg = _process.readShellMsg(); + String command = shellMsg.getCommand(); if (command.equals("sync")) { return; } else if (command.equals("log")) { - String msg = emission.getMsg(); + String msg = shellMsg.getMsg(); LOG.info("Shell msg: " + msg); } else if (command.equals("emit")) { - String stream = emission.getStream(); - Long task = emission.getTask(); - List tuple = emission.getTuple(); - Object messageId = emission.getId(); + String stream = shellMsg.getStream(); + Long task = shellMsg.getTask(); + List tuple = shellMsg.getTuple(); + Object messageId = shellMsg.getId(); if (task == 0) { List outtasks = _collector.emit(stream, tuple, messageId); _process.writeTaskIds(outtasks); } else { - _collector.emitDirect((int)task.longValue(), stream, tuple, messageId); + _collector.emitDirect((int) task.longValue(), stream, + tuple, messageId); } } } diff --git a/storm-core/src/jvm/backtype/storm/task/ShellBolt.java b/storm-core/src/jvm/backtype/storm/task/ShellBolt.java index 36585e3316..6a6c30520c 100644 --- a/storm-core/src/jvm/backtype/storm/task/ShellBolt.java +++ b/storm-core/src/jvm/backtype/storm/task/ShellBolt.java @@ -3,10 +3,9 @@ import backtype.storm.generated.ShellComponent; import backtype.storm.tuple.Tuple; import backtype.storm.utils.ShellProcess; -import backtype.storm.multilang.Emission; -import backtype.storm.multilang.ISerializer; -import backtype.storm.multilang.Immission; -import backtype.storm.multilang.JsonSerializer; +import backtype.storm.multilang.BoltMsg; +import backtype.storm.multilang.ShellMsg; + import java.util.ArrayList; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -62,23 +61,17 @@ public class ShellBolt implements IBolt { public ShellBolt(ShellComponent component) { this(component.get_execution_command(), component.get_script()); - _process = new ShellProcess(new JsonSerializer(), _command); } public ShellBolt(String... command) { _command = command; - _process = new ShellProcess(new JsonSerializer(), _command); - } - - public ShellBolt(ISerializer serializer, String... command) { - _command = command; - _process = new ShellProcess(serializer, _command); } public void prepare(Map stormConf, TopologyContext context, final OutputCollector collector) { _rand = new Random(); _collector = collector; + _process = new ShellProcess(_command); //subprocesses must send their pid first thing Number subpid = _process.launch(stormConf, context); @@ -89,20 +82,20 @@ public void prepare(Map stormConf, TopologyContext context, public void run() { while (_running) { try { - Emission emission = _process.readEmission(); + ShellMsg shellMsg = _process.readShellMsg(); - String command = emission.getCommand(); + String command = shellMsg.getCommand(); if(command.equals("ack")) { - handleAck(emission.getId()); + handleAck(shellMsg.getId()); } else if (command.equals("fail")) { - handleFail(emission.getId()); + handleFail(shellMsg.getId()); } else if (command.equals("error")) { - handleError(emission.getMsg()); + handleError(shellMsg.getMsg()); } else if (command.equals("log")) { - String msg = emission.getMsg(); + String msg = shellMsg.getMsg(); LOG.info("Shell msg: " + msg); } else if (command.equals("emit")) { - handleEmit(emission); + handleEmit(shellMsg); } } catch (InterruptedException e) { } catch (Throwable t) { @@ -118,12 +111,13 @@ public void run() { public void run() { while (_running) { try { - // FIXME: This can either be TaskIds or Immissions Object write = _pendingWrites.poll(1, SECONDS); - if (write instanceof Immission) { - _process.writeImmission((Immission)write); + if (write instanceof BoltMsg) { + _process.writeBoltMsg((BoltMsg)write); } else if (write instanceof List) { - + _process.writeTaskIds((List)write); + } else { + throw new RuntimeException("Cannot write object to bolt:\n" + write.toString()); } } catch (InterruptedException e) { } catch (Throwable t) { @@ -145,14 +139,14 @@ public void execute(Tuple input) { String genId = Long.toString(_rand.nextLong()); _inputs.put(genId, input); try { - Immission immission = new Immission(); - immission.setId(genId); - immission.setComp(input.getSourceComponent()); - immission.setStream(input.getSourceStreamId()); - immission.setTask(input.getSourceTask()); - immission.setTuple(input.getValues()); - - _pendingWrites.put(immission); + BoltMsg boltMsg = new BoltMsg(); + boltMsg.setId(genId); + boltMsg.setComp(input.getSourceComponent()); + boltMsg.setStream(input.getSourceStreamId()); + boltMsg.setTask(input.getSourceTask()); + boltMsg.setTuple(input.getValues()); + + _pendingWrites.put(boltMsg); } catch(InterruptedException e) { throw new RuntimeException("Error during multilang processing", e); } @@ -184,9 +178,9 @@ private void handleError(String msg) { _collector.reportError(new Exception("Shell Process Exception: " + msg)); } - private void handleEmit(Emission emission) throws InterruptedException { + private void handleEmit(ShellMsg shellMsg) throws InterruptedException { List anchors = new ArrayList(); - for (String anchor : emission.getAnchors()) { + for (String anchor : shellMsg.getAnchors()) { Tuple t = _inputs.get(anchor); if (t == null) { throw new RuntimeException("Anchored onto " + anchor + " after ack/fail"); @@ -194,11 +188,12 @@ private void handleEmit(Emission emission) throws InterruptedException { anchors.add(t); } - if(emission.getTask() == 0) { - List outtasks = _collector.emit(emission.getStream(), anchors, emission.getTuple()); + if(shellMsg.getTask() == 0) { + List outtasks = _collector.emit(shellMsg.getStream(), anchors, shellMsg.getTuple()); _pendingWrites.put(outtasks); } else { - _collector.emitDirect((int)emission.getTask(), emission.getStream(), anchors, emission.getTuple()); + _collector.emitDirect((int) shellMsg.getTask(), + shellMsg.getStream(), anchors, shellMsg.getTuple()); } } diff --git a/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java b/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java index 38976bcb59..8871a77b9f 100644 --- a/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java +++ b/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java @@ -1,9 +1,10 @@ package backtype.storm.utils; -import backtype.storm.multilang.Emission; +import backtype.storm.Config; import backtype.storm.multilang.ISerializer; -import backtype.storm.multilang.Immission; +import backtype.storm.multilang.BoltMsg; import backtype.storm.multilang.NoOutputException; +import backtype.storm.multilang.ShellMsg; import backtype.storm.multilang.SpoutMsg; import backtype.storm.task.TopologyContext; import java.io.File; @@ -18,85 +19,109 @@ public class ShellProcess implements Serializable { public static Logger LOG = Logger.getLogger(ShellProcess.class); - private Process _subprocess; - private InputStream processErrorStream; - private String[] command; - public ISerializer serializer; + private Process _subprocess; + private InputStream processErrorStream; + private String[] command; + public ISerializer serializer; - public ShellProcess(ISerializer serializer, String[] command) { + public ShellProcess(String[] command) { this.command = command; - this.serializer = serializer; } public Number launch(Map conf, TopologyContext context) { ProcessBuilder builder = new ProcessBuilder(command); builder.directory(new File(context.getCodeDir())); + this.serializer = getSerializer(conf); + Number pid; try { - _subprocess = builder.start(); - processErrorStream = _subprocess.getErrorStream(); - serializer.initialize(_subprocess.getOutputStream(), _subprocess.getInputStream()); - pid = serializer.connect(conf, context); + _subprocess = builder.start(); + processErrorStream = _subprocess.getErrorStream(); + serializer.initialize(_subprocess.getOutputStream(), + _subprocess.getInputStream()); + pid = serializer.connect(conf, context); } catch (IOException e) { - throw new RuntimeException("Error when launching multilang subprocess\n" + getErrorsString(), e); + throw new RuntimeException( + "Error when launching multilang subprocess\n" + + getErrorsString(), e); } catch (NoOutputException e) { throw new RuntimeException(e + getErrorsString() + "\n"); - } + } return pid; } + private ISerializer getSerializer(Map conf) { + //get factory class name + String serializer_className = (String)conf.get(Config.STORM_MULTILANG_SERIALIZER); + LOG.info("Storm multilang serializer:" + serializer_className); + + ISerializer serializer = null; + try { + //create a factory class + Class klass = Class.forName(serializer_className); + //obtain a serializer object + Object obj = klass.newInstance(); + serializer = (ISerializer)obj; + } catch(Exception e) { + throw new RuntimeException("Failed to construct multilang serializer from serializer " + serializer_className, e); + } + return serializer; + } + public void destroy() { _subprocess.destroy(); } - public Emission readEmission() throws IOException { - try { - return serializer.readEmission(); - } catch (NoOutputException e) { + public ShellMsg readShellMsg() throws IOException { + try { + return serializer.readShellMsg(); + } catch (NoOutputException e) { throw new RuntimeException(e + getErrorsString() + "\n"); - } + } } - public void writeImmission(Immission msg) throws IOException { - serializer.writeImmission(msg); - // drain the error stream to avoid dead lock because of full error stream buffer + public void writeBoltMsg(BoltMsg msg) throws IOException { + serializer.writeBoltMsg(msg); + // drain the error stream to avoid dead lock because of full error + // stream buffer drainErrorStream(); } public void writeSpoutMsg(SpoutMsg msg) throws IOException { - serializer.writeSpoutMsg(msg); - // drain the error stream to avoid dead lock because of full error stream buffer + serializer.writeSpoutMsg(msg); + // drain the error stream to avoid dead lock because of full error + // stream buffer drainErrorStream(); } public void writeTaskIds(List taskIds) throws IOException { - serializer.writeTaskIds(taskIds); - // drain the error stream to avoid dead lock because of full error stream buffer + serializer.writeTaskIds(taskIds); + // drain the error stream to avoid dead lock because of full error + // stream buffer drainErrorStream(); } - public void drainErrorStream() - { + public void drainErrorStream() { try { - while (processErrorStream.available() > 0) - { + while (processErrorStream.available() > 0) { int bufferSize = processErrorStream.available(); - byte[] errorReadingBuffer = new byte[bufferSize]; + byte[] errorReadingBuffer = new byte[bufferSize]; processErrorStream.read(errorReadingBuffer, 0, bufferSize); - LOG.info("Got error from shell process: " + new String(errorReadingBuffer)); + LOG.info("Got error from shell process: " + + new String(errorReadingBuffer)); } - } catch(Exception e) { + } catch (Exception e) { } } public String getErrorsString() { - if(processErrorStream!=null) { + if (processErrorStream != null) { try { return IOUtils.toString(processErrorStream); - } catch(IOException e) { + } catch (IOException e) { return "(Unable to capture error stream)"; } } else { From ee0678c7f8bed531a8514166c7254a2190f3c4a9 Mon Sep 17 00:00:00 2001 From: John Gilmore Date: Tue, 8 Oct 2013 13:52:00 +0200 Subject: [PATCH 223/349] multilang serializer is now handled by stormConf. Renamed Emission and Immission to ShellMsg and BoltMsg respectively. Documented ISerializer interface. --- conf/defaults.yaml | 1 + storm-core/src/jvm/backtype/storm/Config.java | 129 ++++++++-------- .../jvm/backtype/storm/multilang/BoltMsg.java | 63 ++++++++ .../backtype/storm/multilang/Emission.java | 79 ---------- .../backtype/storm/multilang/ISerializer.java | 59 ++++++-- .../backtype/storm/multilang/Immission.java | 52 ------- .../storm/multilang/JsonSerializer.java | 138 +++++++++--------- .../storm/multilang/NoOutputException.java | 19 ++- .../backtype/storm/multilang/ShellMsg.java | 96 ++++++++++++ .../backtype/storm/multilang/SpoutMsg.java | 44 +++--- .../jvm/backtype/storm/spout/ShellSpout.java | 30 ++-- .../jvm/backtype/storm/task/ShellBolt.java | 65 ++++----- .../backtype/storm/utils/ShellProcess.java | 95 +++++++----- 13 files changed, 489 insertions(+), 381 deletions(-) create mode 100644 storm-core/src/jvm/backtype/storm/multilang/BoltMsg.java delete mode 100644 storm-core/src/jvm/backtype/storm/multilang/Emission.java delete mode 100644 storm-core/src/jvm/backtype/storm/multilang/Immission.java create mode 100644 storm-core/src/jvm/backtype/storm/multilang/ShellMsg.java diff --git a/conf/defaults.yaml b/conf/defaults.yaml index b17e445a2b..92e65d6d19 100644 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@ -19,6 +19,7 @@ storm.cluster.mode: "distributed" # can be distributed or local storm.local.mode.zmq: false storm.thrift.transport: "backtype.storm.security.auth.SimpleTransportPlugin" storm.messaging.transport: "backtype.storm.messaging.zmq" +storm.multilang.serializer: "backtype.storm.multilang.JsonSerializer" ### nimbus.* configs are for the master nimbus.host: "localhost" diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java index 564ad0d330..fff157363e 100644 --- a/storm-core/src/jvm/backtype/storm/Config.java +++ b/storm-core/src/jvm/backtype/storm/Config.java @@ -10,11 +10,11 @@ import java.util.Map; /** - * Topology configs are specified as a plain old map. This class provides a - * convenient way to create a topology config map by providing setter methods for - * all the configs that can be set. It also makes it easier to do things like add + * Topology configs are specified as a plain old map. This class provides a + * convenient way to create a topology config map by providing setter methods for + * all the configs that can be set. It also makes it easier to do things like add * serializations. - * + * *

This class also provides constants for all the configurations possible on * a Storm cluster and Storm topology. Each constant is paired with a schema * that defines the validity criterion of the corresponding field. Default @@ -22,10 +22,17 @@ * *

Note that you may put other configurations in any of the configs. Storm * will ignore anything it doesn't recognize, but your topologies are free to make - * use of them by reading them in the prepare method of Bolts or the open method of + * use of them by reading them in the prepare method of Bolts or the open method of * Spouts.

*/ public class Config extends HashMap { + /** + * The serializer for communication between shell components and non-JVM + * processes + */ + public static final String STORM_MULTILANG_SERIALIZER = "storm.multilang.serializer"; + public static final Object STORM_MULTILANG_SERIALIZER_SCHEMA = String.class; + /** * The transporter for communication among Storm tasks */ @@ -35,25 +42,25 @@ public class Config extends HashMap { /** * Netty based messaging: The buffer size for send/recv buffer */ - public static final String STORM_MESSAGING_NETTY_BUFFER_SIZE = "storm.messaging.netty.buffer_size"; + public static final String STORM_MESSAGING_NETTY_BUFFER_SIZE = "storm.messaging.netty.buffer_size"; public static final Object STORM_MESSAGING_NETTY_BUFFER_SIZE_SCHEMA = Number.class; /** * Netty based messaging: The max # of retries that a peer will perform when a remote is not accessible */ - public static final String STORM_MESSAGING_NETTY_MAX_RETRIES = "storm.messaging.netty.max_retries"; + public static final String STORM_MESSAGING_NETTY_MAX_RETRIES = "storm.messaging.netty.max_retries"; public static final Object STORM_MESSAGING_NETTY_MAX_RETRIES_SCHEMA = Number.class; /** - * Netty based messaging: The min # of milliseconds that a peer will wait. + * Netty based messaging: The min # of milliseconds that a peer will wait. */ - public static final String STORM_MESSAGING_NETTY_MIN_SLEEP_MS = "storm.messaging.netty.min_wait_ms"; + public static final String STORM_MESSAGING_NETTY_MIN_SLEEP_MS = "storm.messaging.netty.min_wait_ms"; public static final Object STORM_MESSAGING_NETTY_MIN_SLEEP_MS_SCHEMA = Number.class; /** - * Netty based messaging: The max # of milliseconds that a peer will wait. + * Netty based messaging: The max # of milliseconds that a peer will wait. */ - public static final String STORM_MESSAGING_NETTY_MAX_SLEEP_MS = "storm.messaging.netty.max_wait_ms"; + public static final String STORM_MESSAGING_NETTY_MAX_SLEEP_MS = "storm.messaging.netty.max_wait_ms"; public static final Object STORM_MESSAGING_NETTY_MAX_SLEEP_MS_SCHEMA = Number.class; /** @@ -78,7 +85,7 @@ public class Config extends HashMap { /** * A global task scheduler used to assign topologies's tasks to supervisors' wokers. - * + * * If this is not set, a default system scheduler will be used. */ public static final String STORM_SCHEDULER = "storm.scheduler"; @@ -91,9 +98,9 @@ public class Config extends HashMap { public static final Object STORM_CLUSTER_MODE_SCHEMA = String.class; /** - * The hostname the supervisors/workers should report to nimbus. If unset, Storm will + * The hostname the supervisors/workers should report to nimbus. If unset, Storm will * get the hostname to report by calling InetAddress.getLocalHost().getCanonicalHostName(). - * + * * You should set this config when you dont have a DNS which supervisors/workers * can utilize to find each other based on hostname got from calls to * InetAddress.getLocalHost().getCanonicalHostName(). @@ -108,16 +115,16 @@ public class Config extends HashMap { public static final Object STORM_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class; /** - * The serializer class for ListDelegate (tuple payload). + * The serializer class for ListDelegate (tuple payload). * The default serializer will be ListDelegateSerializer */ public static final String TOPOLOGY_TUPLE_SERIALIZER = "topology.tuple.serializer"; public static final Object TOPOLOGY_TUPLE_SERIALIZER_SCHEMA = String.class; /** - * Whether or not to use ZeroMQ for messaging in local mode. If this is set - * to false, then Storm will use a pure-Java messaging system. The purpose - * of this flag is to make it easy to run Storm in local mode by eliminating + * Whether or not to use ZeroMQ for messaging in local mode. If this is set + * to false, then Storm will use a pure-Java messaging system. The purpose + * of this flag is to make it easy to run Storm in local mode by eliminating * the need for native dependencies, which can be difficult to install. * * Defaults to false. @@ -255,7 +262,7 @@ public class Config extends HashMap { public static final Object NIMBUS_TASK_LAUNCH_SECS_SCHEMA = Number.class; /** - * Whether or not nimbus should reassign tasks if it detects that a task goes down. + * Whether or not nimbus should reassign tasks if it detects that a task goes down. * Defaults to true, and it's not recommended to change this value. */ public static final String NIMBUS_REASSIGN = "nimbus.reassign"; @@ -319,19 +326,19 @@ public class Config extends HashMap { public static final Object DRPC_PORT_SCHEMA = Number.class; /** - * DRPC thrift server worker threads + * DRPC thrift server worker threads */ public static final String DRPC_WORKER_THREADS = "drpc.worker.threads"; public static final Object DRPC_WORKER_THREADS_SCHEMA = Number.class; /** - * DRPC thrift server queue size + * DRPC thrift server queue size */ public static final String DRPC_QUEUE_SIZE = "drpc.queue.size"; public static final Object DRPC_QUEUE_SIZE_SCHEMA = Number.class; /** - * This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back. + * This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back. */ public static final String DRPC_INVOCATIONS_PORT = "drpc.invocations.port"; public static final Object DRPC_INVOCATIONS_PORT_SCHEMA = Number.class; @@ -479,8 +486,8 @@ public class Config extends HashMap { /** * How many instances to create for a spout/bolt. A task runs on a thread with zero or more * other tasks for the same spout/bolt. The number of tasks for a spout/bolt is always - * the same throughout the lifetime of a topology, but the number of executors (threads) for - * a spout/bolt can change over time. This allows a topology to scale to more or less resources + * the same throughout the lifetime of a topology, but the number of executors (threads) for + * a spout/bolt can change over time. This allows a topology to scale to more or less resources * without redeploying the topology or violating the constraints of Storm (such as a fields grouping * guaranteeing that the same value goes to the same task). */ @@ -519,8 +526,8 @@ public class Config extends HashMap { /** * A list of classes that customize storm's kryo instance during start-up. - * Each listed class name must implement IKryoDecorator. During start-up the - * listed class is instantiated with 0 arguments, then its 'decorate' method + * Each listed class name must implement IKryoDecorator. During start-up the + * listed class is instantiated with 0 arguments, then its 'decorate' method * is called with storm's kryo instance as the only argument. */ public static final String TOPOLOGY_KRYO_DECORATORS = "topology.kryo.decorators"; @@ -550,7 +557,7 @@ public class Config extends HashMap { /* * A list of classes implementing IMetricsConsumer (See storm.yaml.example for exact config format). - * Each listed class will be routed all the metrics data generated by the storm metrics API. + * Each listed class will be routed all the metrics data generated by the storm metrics API. * Each listed class maps 1:1 to a system bolt named __metrics_ClassName#N, and it's parallelism is configurable. */ public static final String TOPOLOGY_METRICS_CONSUMER_REGISTER = "topology.metrics.consumer.register"; @@ -566,24 +573,24 @@ public class Config extends HashMap { /** - * The maximum number of tuples that can be pending on a spout task at any given time. - * This config applies to individual tasks, not to spouts or topologies as a whole. - * + * The maximum number of tuples that can be pending on a spout task at any given time. + * This config applies to individual tasks, not to spouts or topologies as a whole. + * * A pending tuple is one that has been emitted from a spout but has not been acked or failed yet. - * Note that this config parameter has no effect for unreliable spouts that don't tag + * Note that this config parameter has no effect for unreliable spouts that don't tag * their tuples with a message id. */ - public static final String TOPOLOGY_MAX_SPOUT_PENDING="topology.max.spout.pending"; + public static final String TOPOLOGY_MAX_SPOUT_PENDING="topology.max.spout.pending"; public static final Object TOPOLOGY_MAX_SPOUT_PENDING_SCHEMA = Number.class; /** * A class that implements a strategy for what to do when a spout needs to wait. Waiting is * triggered in one of two conditions: - * + * * 1. nextTuple emits no tuples * 2. The spout has hit maxSpoutPending and can't emit any more tuples */ - public static final String TOPOLOGY_SPOUT_WAIT_STRATEGY="topology.spout.wait.strategy"; + public static final String TOPOLOGY_SPOUT_WAIT_STRATEGY="topology.spout.wait.strategy"; public static final Object TOPOLOGY_SPOUT_WAIT_STRATEGY_SCHEMA = String.class; /** @@ -606,7 +613,7 @@ public class Config extends HashMap { public static final Object TOPOLOGY_STATS_SAMPLE_RATE_SCHEMA = Number.class; /** - * The time period that builtin metrics data in bucketed into. + * The time period that builtin metrics data in bucketed into. */ public static final String TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS="topology.builtin.metrics.bucket.size.secs"; public static final Object TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS_SCHEMA = Number.class; @@ -633,7 +640,7 @@ public class Config extends HashMap { /** * A list of task hooks that are automatically added to every spout and bolt in the topology. An example - * of when you'd do this is to add a hook that integrates with your internal + * of when you'd do this is to add a hook that integrates with your internal * monitoring system. These hooks are instantiated using the zero-arg constructor. */ public static final String TOPOLOGY_AUTO_TASK_HOOKS="topology.auto.task.hooks"; @@ -647,7 +654,7 @@ public class Config extends HashMap { public static final Object TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE_SCHEMA = ConfigValidation.PowerOf2Validator; /** - * The maximum number of messages to batch from the thread receiving off the network to the + * The maximum number of messages to batch from the thread receiving off the network to the * executor queues. Must be a power of 2. */ public static final String TOPOLOGY_RECEIVER_BUFFER_SIZE="topology.receiver.buffer.size"; @@ -681,14 +688,14 @@ public class Config extends HashMap { public static final Object TOPOLOGY_DISRUPTOR_WAIT_STRATEGY_SCHEMA = String.class; /** - * The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed + * The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed * via the TopologyContext. */ public static final String TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE="topology.worker.shared.thread.pool.size"; public static final Object TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE_SCHEMA = Number.class; /** - * The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example, + * The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example, * an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be * reported to Zookeeper per task for every 10 second interval of time. */ @@ -757,9 +764,9 @@ public class Config extends HashMap { /** * This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers) - * for the java.library.path value. java.library.path tells the JVM where + * for the java.library.path value. java.library.path tells the JVM where * to look for native libraries. It is necessary to set this config correctly since - * Storm uses the ZeroMQ and JZMQ native libs. + * Storm uses the ZeroMQ and JZMQ native libs. */ public static final String JAVA_LIBRARY_PATH = "java.library.path"; public static final Object JAVA_LIBRARY_PATH_SCHEMA = String.class; @@ -781,17 +788,17 @@ public class Config extends HashMap { public static void setDebug(Map conf, boolean isOn) { conf.put(Config.TOPOLOGY_DEBUG, isOn); - } + } public void setDebug(boolean isOn) { setDebug(this, isOn); } - + @Deprecated public void setOptimize(boolean isOn) { put(Config.TOPOLOGY_OPTIMIZE, isOn); - } - + } + public static void setNumWorkers(Map conf, int workers) { conf.put(Config.TOPOLOGY_WORKERS, workers); } @@ -807,7 +814,7 @@ public static void setNumAckers(Map conf, int numExecutors) { public void setNumAckers(int numExecutors) { setNumAckers(this, numExecutors); } - + public static void setMessageTimeoutSecs(Map conf, int secs) { conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, secs); } @@ -815,7 +822,7 @@ public static void setMessageTimeoutSecs(Map conf, int secs) { public void setMessageTimeoutSecs(int secs) { setMessageTimeoutSecs(this, secs); } - + public static void registerSerialization(Map conf, Class klass) { getRegisteredSerializations(conf).add(klass.getName()); } @@ -823,17 +830,17 @@ public static void registerSerialization(Map conf, Class klass) { public void registerSerialization(Class klass) { registerSerialization(this, klass); } - + public static void registerSerialization(Map conf, Class klass, Class serializerClass) { Map register = new HashMap(); register.put(klass.getName(), serializerClass.getName()); - getRegisteredSerializations(conf).add(register); + getRegisteredSerializations(conf).add(register); } public void registerSerialization(Class klass, Class serializerClass) { registerSerialization(this, klass, serializerClass); } - + public void registerMetricsConsumer(Class klass, Object argument, long parallelismHint) { HashMap m = new HashMap(); m.put("class", klass.getCanonicalName()); @@ -861,7 +868,7 @@ public static void registerDecorator(Map conf, Class k public void registerDecorator(Class klass) { registerDecorator(this, klass); } - + public static void setKryoFactory(Map conf, Class klass) { conf.put(Config.TOPOLOGY_KRYO_FACTORY, klass.getName()); } @@ -877,7 +884,7 @@ public static void setSkipMissingKryoRegistrations(Map conf, boolean skip) { public void setSkipMissingKryoRegistrations(boolean skip) { setSkipMissingKryoRegistrations(this, skip); } - + public static void setMaxTaskParallelism(Map conf, int max) { conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, max); } @@ -885,7 +892,7 @@ public static void setMaxTaskParallelism(Map conf, int max) { public void setMaxTaskParallelism(int max) { setMaxTaskParallelism(this, max); } - + public static void setMaxSpoutPending(Map conf, int max) { conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, max); } @@ -893,23 +900,23 @@ public static void setMaxSpoutPending(Map conf, int max) { public void setMaxSpoutPending(int max) { setMaxSpoutPending(this, max); } - + public static void setStatsSampleRate(Map conf, double rate) { conf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, rate); - } + } public void setStatsSampleRate(double rate) { setStatsSampleRate(this, rate); - } + } public static void setFallBackOnJavaSerialization(Map conf, boolean fallback) { conf.put(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION, fallback); - } + } public void setFallBackOnJavaSerialization(boolean fallback) { setFallBackOnJavaSerialization(this, fallback); - } - + } + private static List getRegisteredSerializations(Map conf) { List ret; if(!conf.containsKey(Config.TOPOLOGY_KRYO_REGISTER)) { @@ -920,13 +927,13 @@ private static List getRegisteredSerializations(Map conf) { conf.put(Config.TOPOLOGY_KRYO_REGISTER, ret); return ret; } - + private static List getRegisteredDecorators(Map conf) { List ret; if(!conf.containsKey(Config.TOPOLOGY_KRYO_DECORATORS)) { ret = new ArrayList(); } else { - ret = new ArrayList((List) conf.get(Config.TOPOLOGY_KRYO_DECORATORS)); + ret = new ArrayList((List) conf.get(Config.TOPOLOGY_KRYO_DECORATORS)); } conf.put(Config.TOPOLOGY_KRYO_DECORATORS, ret); return ret; diff --git a/storm-core/src/jvm/backtype/storm/multilang/BoltMsg.java b/storm-core/src/jvm/backtype/storm/multilang/BoltMsg.java new file mode 100644 index 0000000000..1d6bd1d9e7 --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/multilang/BoltMsg.java @@ -0,0 +1,63 @@ +package backtype.storm.multilang; + +import java.util.List; + +/** + * BoltMsg is an object that represents the data sent from a shell component to + * a bolt process that implements a multi-language protocol. It is the union of + * all data types that a bolt can receive from Storm. + * + *

+ * BoltMsgs are objects sent to the ISerializer interface, for serialization + * according to the wire protocol implemented by the serializer. The BoltMsg + * class allows for a decoupling between the serialized representation of the + * data and the data itself. + *

+ */ +public class BoltMsg { + private String id; + private String comp; + private String stream; + private long task; + private List tuple; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getComp() { + return comp; + } + + public void setComp(String comp) { + this.comp = comp; + } + + public String getStream() { + return stream; + } + + public void setStream(String stream) { + this.stream = stream; + } + + public long getTask() { + return task; + } + + public void setTask(long task) { + this.task = task; + } + + public List getTuple() { + return tuple; + } + + public void setTuple(List tuple) { + this.tuple = tuple; + } +} diff --git a/storm-core/src/jvm/backtype/storm/multilang/Emission.java b/storm-core/src/jvm/backtype/storm/multilang/Emission.java deleted file mode 100644 index d477d8b6eb..0000000000 --- a/storm-core/src/jvm/backtype/storm/multilang/Emission.java +++ /dev/null @@ -1,79 +0,0 @@ -package backtype.storm.multilang; - -import java.util.ArrayList; -import java.util.List; - -/** - * Emission is an object that represents the data sent to a shell component - * from a process that implements a multi-language protocol. It is the union - * of all data types that a component can send to Storm. - * - *

Emissions are objects received from the ISerializer interface, after - * the serializer has deserialized the data from the underlying wire protocol. - * The Emission class allows for a decoupling between the serialized - * representation of the data and the data itself.

- */ -public class Emission { - private String command; - private String id; - private List anchors; - private String stream; - private long task; - private String msg; - private List tuple; - - public String getCommand() { - return command; - } - public void setCommand(String command) { - this.command = command; - } - public String getId() { - return id; - } - public void setId(String id) { - this.id = id; - } - public List getAnchors() { - return anchors; - } - public void setAnchors(List anchors) { - this.anchors = anchors; - } - public void addAnchor(String anchor) { - if (anchors == null) { - anchors = new ArrayList(); - } - this.anchors.add(anchor); - } - public String getStream() { - return stream; - } - public void setStream(String stream) { - this.stream = stream; - } - public long getTask() { - return task; - } - public void setTask(long task) { - this.task = task; - } - public String getMsg() { - return msg; - } - public void setMsg(String msg) { - this.msg = msg; - } - public List getTuple() { - return tuple; - } - public void setTuple(List tuple) { - this.tuple = tuple; - } - public void addTuple(Object tuple) { - if (this.tuple == null) { - this.tuple = new ArrayList(); - } - this.tuple.add(tuple); - } -} diff --git a/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java b/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java index 691b432e32..a6ccfd4326 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java +++ b/storm-core/src/jvm/backtype/storm/multilang/ISerializer.java @@ -10,15 +10,56 @@ import backtype.storm.task.TopologyContext; /** - * The ISerializer interface describes the methods that an object should implement - * to provide serialization and de-serialization capabilities to non-JVM - * language components. + * The ISerializer interface describes the methods that an object should + * implement to provide serialization and de-serialization capabilities to + * non-JVM language components. */ public interface ISerializer extends Serializable { - void initialize (OutputStream processIn, InputStream processOut); - Number connect (Map conf, TopologyContext context) throws IOException, NoOutputException; - Emission readEmission() throws IOException, NoOutputException; - void writeImmission(Immission immission) throws IOException; - void writeSpoutMsg(SpoutMsg msg) throws IOException; - void writeTaskIds(List taskIds) throws IOException; + + /** + * This method sets the input and output streams of the serializer + * + * @param processIn output stream to non-JVM component + * @param processOut input stream from non-JVM component + */ + void initialize(OutputStream processIn, InputStream processOut); + + /** + * This method transmits the Storm config to the non-JVM process and + * receives its pid. + * + * @param conf storm configuration + * @param context topology context + * @return process pid + */ + Number connect(Map conf, TopologyContext context) throws IOException, + NoOutputException; + + /** + * This method receives a shell message from the non-JVM process + * + * @return shell message + */ + ShellMsg readShellMsg() throws IOException, NoOutputException; + + /** + * This method sends a bolt message to a non-JVM bolt process + * + * @param msg bolt message + */ + void writeBoltMsg(BoltMsg msg) throws IOException; + + /** + * This method sends a spout message to a non-JVM spout process + * + * @param msg spout message + */ + void writeSpoutMsg(SpoutMsg msg) throws IOException; + + /** + * This method sends a list of task IDs to a non-JVM bolt process + * + * @param taskIds list of task IDs + */ + void writeTaskIds(List taskIds) throws IOException; } diff --git a/storm-core/src/jvm/backtype/storm/multilang/Immission.java b/storm-core/src/jvm/backtype/storm/multilang/Immission.java deleted file mode 100644 index 968f7286c6..0000000000 --- a/storm-core/src/jvm/backtype/storm/multilang/Immission.java +++ /dev/null @@ -1,52 +0,0 @@ -package backtype.storm.multilang; - -import java.util.List; - -/** - * Immission is an object that represents the data sent from a shell component - * to a process that implements a multi-language protocol. It is the union - * of all data types that a component can receive from Storm. - * - *

Immissions are objects sent to the ISerializer interface, for - * serialization according to the wire protocol implemented by the serializer. - * The Immission class allows for a decoupling between the serialized - * representation of the data and the data itself.

- */ -public class Immission { - private String id; - private String comp; - private String stream; - private long task; - private List tuple; - - public String getId() { - return id; - } - public void setId(String id) { - this.id = id; - } - public String getComp() { - return comp; - } - public void setComp(String comp) { - this.comp = comp; - } - public String getStream() { - return stream; - } - public void setStream(String stream) { - this.stream = stream; - } - public long getTask() { - return task; - } - public void setTask(long task) { - this.task = task; - } - public List getTuple() { - return tuple; - } - public void setTuple(List tuple) { - this.tuple = tuple; - } -} diff --git a/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java b/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java index ba574e577e..ceb8cb3ed4 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java +++ b/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java @@ -22,126 +22,128 @@ * JsonSerializer implements the JSON multilang protocol. */ public class JsonSerializer implements ISerializer { - private DataOutputStream processIn; + private DataOutputStream processIn; private BufferedReader processOut; - public void initialize (OutputStream processIn, InputStream processOut) { - this.processIn = new DataOutputStream(processIn);; - this.processOut = new BufferedReader(new InputStreamReader(processOut)); - } + public void initialize(OutputStream processIn, InputStream processOut) { + this.processIn = new DataOutputStream(processIn); + this.processOut = new BufferedReader(new InputStreamReader(processOut)); + } - public Number connect (Map conf, TopologyContext context) throws IOException, NoOutputException { - JSONObject setupInfo = new JSONObject(); + public Number connect(Map conf, TopologyContext context) + throws IOException, NoOutputException { + JSONObject setupInfo = new JSONObject(); setupInfo.put("pidDir", context.getPIDDir()); setupInfo.put("conf", conf); setupInfo.put("context", context); - writeMessage(setupInfo); + writeMessage(setupInfo); - Number pid = (Number)((JSONObject)readMessage()).get("pid"); + Number pid = (Number) ((JSONObject) readMessage()).get("pid"); return pid; - } - - public void writeImmission(Immission immission) throws IOException { - JSONObject obj = new JSONObject(); - obj.put("id", immission.getId()); - obj.put("comp", immission.getComp()); - obj.put("stream", immission.getStream()); - obj.put("task", immission.getTask()); - obj.put("tuple", immission.getTuple()); + } + + public void writeBoltMsg(BoltMsg boltMsg) throws IOException { + JSONObject obj = new JSONObject(); + obj.put("id", boltMsg.getId()); + obj.put("comp", boltMsg.getComp()); + obj.put("stream", boltMsg.getStream()); + obj.put("task", boltMsg.getTask()); + obj.put("tuple", boltMsg.getTuple()); writeMessage(obj); - } + } - public void writeSpoutMsg(SpoutMsg msg) throws IOException { - JSONObject obj = new JSONObject(); + public void writeSpoutMsg(SpoutMsg msg) throws IOException { + JSONObject obj = new JSONObject(); obj.put("command", msg.getCommand()); obj.put("id", msg.getId()); writeMessage(obj); - } + } - public void writeTaskIds(List taskIds) throws IOException { + public void writeTaskIds(List taskIds) throws IOException { writeMessage(taskIds); - } + } - private void writeMessage(Object msg) throws IOException { + private void writeMessage(Object msg) throws IOException { writeString(JSONValue.toJSONString(msg)); } - private void writeString(String str) throws IOException { + private void writeString(String str) throws IOException { byte[] strBytes = str.getBytes("UTF-8"); processIn.write(strBytes, 0, strBytes.length); processIn.writeBytes("\nend\n"); processIn.flush(); } - public Emission readEmission() throws IOException, NoOutputException { - JSONObject msg = (JSONObject)readMessage(); - Emission emission = new Emission(); + public ShellMsg readShellMsg() throws IOException, NoOutputException { + JSONObject msg = (JSONObject) readMessage(); + ShellMsg shellMsg = new ShellMsg(); - String command = (String) msg.get("command"); - emission.setCommand(command); + String command = (String) msg.get("command"); + shellMsg.setCommand(command); - String stream = (String) msg.get("stream"); - if(stream == null) stream = Utils.DEFAULT_STREAM_ID; - emission.setStream(stream); + String stream = (String) msg.get("stream"); + if (stream == null) + stream = Utils.DEFAULT_STREAM_ID; + shellMsg.setStream(stream); Object taskObj = msg.get("task"); if (taskObj != null) { - emission.setTask((Long) taskObj); + shellMsg.setTask((Long) taskObj); } - emission.setTuple((List) msg.get("tuple")); + shellMsg.setTuple((List) msg.get("tuple")); List anchors = new ArrayList(); Object anchorObj = msg.get("anchors"); - if(anchorObj!=null) { - if(anchorObj instanceof String) { + if (anchorObj != null) { + if (anchorObj instanceof String) { anchorObj = Arrays.asList(anchorObj); } - for(Object o: (List) anchorObj) { - emission.addAnchor((String)o); + for (Object o : (List) anchorObj) { + shellMsg.addAnchor((String) o); } } - return emission; + return shellMsg; } - private Object readMessage() throws IOException, NoOutputException { - String string = readString(); + private Object readMessage() throws IOException, NoOutputException { + String string = readString(); Object msg = JSONValue.parse(string); if (msg != null) { - return msg; + return msg; } else { - throw new IOException("unable to parse: " + string); + throw new IOException("unable to parse: " + string); } - } + } private String readString() throws IOException, NoOutputException { StringBuilder line = new StringBuilder(); - //synchronized (processOut) { - while (true) { - String subline = processOut.readLine(); - if(subline==null) { - StringBuilder errorMessage = new StringBuilder(); - errorMessage.append("Pipe to subprocess seems to be broken!"); - if (line.length() == 0) { - errorMessage.append(" No output read.\n"); - } - else { - errorMessage.append(" Currently read output: " + line.toString() + "\n"); - } - errorMessage.append("Serializer Exception:\n"); - throw new NoOutputException(errorMessage.toString()); - } - if(subline.equals("end")) { - break; - } - if(line.length()!=0) { - line.append("\n"); + // synchronized (processOut) { + while (true) { + String subline = processOut.readLine(); + if (subline == null) { + StringBuilder errorMessage = new StringBuilder(); + errorMessage.append("Pipe to subprocess seems to be broken!"); + if (line.length() == 0) { + errorMessage.append(" No output read.\n"); + } else { + errorMessage.append(" Currently read output: " + + line.toString() + "\n"); } - line.append(subline); + errorMessage.append("Serializer Exception:\n"); + throw new NoOutputException(errorMessage.toString()); } - //} + if (subline.equals("end")) { + break; + } + if (line.length() != 0) { + line.append("\n"); + } + line.append(subline); + } + // } return line.toString(); } } diff --git a/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java b/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java index 8ffb7dd432..fd5ef72639 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java +++ b/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java @@ -5,8 +5,19 @@ * non-JVM process. */ public class NoOutputException extends Exception { - public NoOutputException() { super(); } - public NoOutputException(String message) { super(message); } - public NoOutputException(String message, Throwable cause) { super(message, cause); } - public NoOutputException(Throwable cause) { super(cause); } + public NoOutputException() { + super(); + } + + public NoOutputException(String message) { + super(message); + } + + public NoOutputException(String message, Throwable cause) { + super(message, cause); + } + + public NoOutputException(Throwable cause) { + super(cause); + } } \ No newline at end of file diff --git a/storm-core/src/jvm/backtype/storm/multilang/ShellMsg.java b/storm-core/src/jvm/backtype/storm/multilang/ShellMsg.java new file mode 100644 index 0000000000..d56ba7854c --- /dev/null +++ b/storm-core/src/jvm/backtype/storm/multilang/ShellMsg.java @@ -0,0 +1,96 @@ +package backtype.storm.multilang; + +import java.util.ArrayList; +import java.util.List; + +/** + * ShellMsg is an object that represents the data sent to a shell component from + * a process that implements a multi-language protocol. It is the union of all + * data types that a component can send to Storm. + * + *

+ * ShellMsgs are objects received from the ISerializer interface, after the + * serializer has deserialized the data from the underlying wire protocol. The + * ShellMsg class allows for a decoupling between the serialized representation + * of the data and the data itself. + *

+ */ +public class ShellMsg { + private String command; + private String id; + private List anchors; + private String stream; + private long task; + private String msg; + private List tuple; + + public String getCommand() { + return command; + } + + public void setCommand(String command) { + this.command = command; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public List getAnchors() { + return anchors; + } + + public void setAnchors(List anchors) { + this.anchors = anchors; + } + + public void addAnchor(String anchor) { + if (anchors == null) { + anchors = new ArrayList(); + } + this.anchors.add(anchor); + } + + public String getStream() { + return stream; + } + + public void setStream(String stream) { + this.stream = stream; + } + + public long getTask() { + return task; + } + + public void setTask(long task) { + this.task = task; + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } + + public List getTuple() { + return tuple; + } + + public void setTuple(List tuple) { + this.tuple = tuple; + } + + public void addTuple(Object tuple) { + if (this.tuple == null) { + this.tuple = new ArrayList(); + } + this.tuple.add(tuple); + } +} diff --git a/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java b/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java index c8063ebf36..9b5ffce91e 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java +++ b/storm-core/src/jvm/backtype/storm/multilang/SpoutMsg.java @@ -1,28 +1,34 @@ package backtype.storm.multilang; /** - * SpoutMsg is an object that represents the data sent from a shell spout - * to a process that implements a multi-language spout. The SpoutMsg is used - * to send a "next", "ack" or "fail" message to a spout. + * SpoutMsg is an object that represents the data sent from a shell spout to a + * process that implements a multi-language spout. The SpoutMsg is used to send + * a "next", "ack" or "fail" message to a spout. * - *

Spout messages are objects sent to the ISerializer interface, for + *

+ * Spout messages are objects sent to the ISerializer interface, for * serialization according to the wire protocol implemented by the serializer. * The SpoutMsg class allows for a decoupling between the serialized - * representation of the data and the data itself.

+ * representation of the data and the data itself. + *

*/ public class SpoutMsg { - private String command; - private String id; - public String getCommand() { - return command; - } - public void setCommand(String command) { - this.command = command; - } - public String getId() { - return id; - } - public void setId(String id) { - this.id = id; - } + private String command; + private String id; + + public String getCommand() { + return command; + } + + public void setCommand(String command) { + this.command = command; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } } diff --git a/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java b/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java index c9ef682b3d..72cebe28c2 100644 --- a/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java +++ b/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java @@ -1,13 +1,10 @@ package backtype.storm.spout; import backtype.storm.generated.ShellComponent; -import backtype.storm.multilang.Emission; -import backtype.storm.multilang.ISerializer; -import backtype.storm.multilang.JsonSerializer; +import backtype.storm.multilang.ShellMsg; import backtype.storm.multilang.SpoutMsg; import backtype.storm.task.TopologyContext; import backtype.storm.utils.ShellProcess; -import backtype.storm.utils.Utils; import java.util.Map; import java.util.List; import java.io.IOException; @@ -25,22 +22,16 @@ public class ShellSpout implements ISpout { public ShellSpout(ShellComponent component) { this(component.get_execution_command(), component.get_script()); - _process = new ShellProcess(new JsonSerializer(), _command); } public ShellSpout(String... command) { _command = command; - _process = new ShellProcess(new JsonSerializer(), _command); - } - - public ShellSpout(ISerializer serializer, String... command) { - _command = command; - _process = new ShellProcess(serializer, _command); } public void open(Map stormConf, TopologyContext context, SpoutOutputCollector collector) { _collector = collector; + _process = new ShellProcess(_command); Number subpid = _process.launch(stormConf, context); LOG.info("Launched subprocess with pid " + subpid); @@ -82,23 +73,24 @@ private void querySubprocess() { _process.writeSpoutMsg(spoutMsg); while (true) { - Emission emission = _process.readEmission(); - String command = emission.getCommand(); + ShellMsg shellMsg = _process.readShellMsg(); + String command = shellMsg.getCommand(); if (command.equals("sync")) { return; } else if (command.equals("log")) { - String msg = emission.getMsg(); + String msg = shellMsg.getMsg(); LOG.info("Shell msg: " + msg); } else if (command.equals("emit")) { - String stream = emission.getStream(); - Long task = emission.getTask(); - List tuple = emission.getTuple(); - Object messageId = emission.getId(); + String stream = shellMsg.getStream(); + Long task = shellMsg.getTask(); + List tuple = shellMsg.getTuple(); + Object messageId = shellMsg.getId(); if (task == 0) { List outtasks = _collector.emit(stream, tuple, messageId); _process.writeTaskIds(outtasks); } else { - _collector.emitDirect((int)task.longValue(), stream, tuple, messageId); + _collector.emitDirect((int) task.longValue(), stream, + tuple, messageId); } } } diff --git a/storm-core/src/jvm/backtype/storm/task/ShellBolt.java b/storm-core/src/jvm/backtype/storm/task/ShellBolt.java index 36585e3316..6a6c30520c 100644 --- a/storm-core/src/jvm/backtype/storm/task/ShellBolt.java +++ b/storm-core/src/jvm/backtype/storm/task/ShellBolt.java @@ -3,10 +3,9 @@ import backtype.storm.generated.ShellComponent; import backtype.storm.tuple.Tuple; import backtype.storm.utils.ShellProcess; -import backtype.storm.multilang.Emission; -import backtype.storm.multilang.ISerializer; -import backtype.storm.multilang.Immission; -import backtype.storm.multilang.JsonSerializer; +import backtype.storm.multilang.BoltMsg; +import backtype.storm.multilang.ShellMsg; + import java.util.ArrayList; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -62,23 +61,17 @@ public class ShellBolt implements IBolt { public ShellBolt(ShellComponent component) { this(component.get_execution_command(), component.get_script()); - _process = new ShellProcess(new JsonSerializer(), _command); } public ShellBolt(String... command) { _command = command; - _process = new ShellProcess(new JsonSerializer(), _command); - } - - public ShellBolt(ISerializer serializer, String... command) { - _command = command; - _process = new ShellProcess(serializer, _command); } public void prepare(Map stormConf, TopologyContext context, final OutputCollector collector) { _rand = new Random(); _collector = collector; + _process = new ShellProcess(_command); //subprocesses must send their pid first thing Number subpid = _process.launch(stormConf, context); @@ -89,20 +82,20 @@ public void prepare(Map stormConf, TopologyContext context, public void run() { while (_running) { try { - Emission emission = _process.readEmission(); + ShellMsg shellMsg = _process.readShellMsg(); - String command = emission.getCommand(); + String command = shellMsg.getCommand(); if(command.equals("ack")) { - handleAck(emission.getId()); + handleAck(shellMsg.getId()); } else if (command.equals("fail")) { - handleFail(emission.getId()); + handleFail(shellMsg.getId()); } else if (command.equals("error")) { - handleError(emission.getMsg()); + handleError(shellMsg.getMsg()); } else if (command.equals("log")) { - String msg = emission.getMsg(); + String msg = shellMsg.getMsg(); LOG.info("Shell msg: " + msg); } else if (command.equals("emit")) { - handleEmit(emission); + handleEmit(shellMsg); } } catch (InterruptedException e) { } catch (Throwable t) { @@ -118,12 +111,13 @@ public void run() { public void run() { while (_running) { try { - // FIXME: This can either be TaskIds or Immissions Object write = _pendingWrites.poll(1, SECONDS); - if (write instanceof Immission) { - _process.writeImmission((Immission)write); + if (write instanceof BoltMsg) { + _process.writeBoltMsg((BoltMsg)write); } else if (write instanceof List) { - + _process.writeTaskIds((List)write); + } else { + throw new RuntimeException("Cannot write object to bolt:\n" + write.toString()); } } catch (InterruptedException e) { } catch (Throwable t) { @@ -145,14 +139,14 @@ public void execute(Tuple input) { String genId = Long.toString(_rand.nextLong()); _inputs.put(genId, input); try { - Immission immission = new Immission(); - immission.setId(genId); - immission.setComp(input.getSourceComponent()); - immission.setStream(input.getSourceStreamId()); - immission.setTask(input.getSourceTask()); - immission.setTuple(input.getValues()); - - _pendingWrites.put(immission); + BoltMsg boltMsg = new BoltMsg(); + boltMsg.setId(genId); + boltMsg.setComp(input.getSourceComponent()); + boltMsg.setStream(input.getSourceStreamId()); + boltMsg.setTask(input.getSourceTask()); + boltMsg.setTuple(input.getValues()); + + _pendingWrites.put(boltMsg); } catch(InterruptedException e) { throw new RuntimeException("Error during multilang processing", e); } @@ -184,9 +178,9 @@ private void handleError(String msg) { _collector.reportError(new Exception("Shell Process Exception: " + msg)); } - private void handleEmit(Emission emission) throws InterruptedException { + private void handleEmit(ShellMsg shellMsg) throws InterruptedException { List anchors = new ArrayList(); - for (String anchor : emission.getAnchors()) { + for (String anchor : shellMsg.getAnchors()) { Tuple t = _inputs.get(anchor); if (t == null) { throw new RuntimeException("Anchored onto " + anchor + " after ack/fail"); @@ -194,11 +188,12 @@ private void handleEmit(Emission emission) throws InterruptedException { anchors.add(t); } - if(emission.getTask() == 0) { - List outtasks = _collector.emit(emission.getStream(), anchors, emission.getTuple()); + if(shellMsg.getTask() == 0) { + List outtasks = _collector.emit(shellMsg.getStream(), anchors, shellMsg.getTuple()); _pendingWrites.put(outtasks); } else { - _collector.emitDirect((int)emission.getTask(), emission.getStream(), anchors, emission.getTuple()); + _collector.emitDirect((int) shellMsg.getTask(), + shellMsg.getStream(), anchors, shellMsg.getTuple()); } } diff --git a/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java b/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java index 38976bcb59..8871a77b9f 100644 --- a/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java +++ b/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java @@ -1,9 +1,10 @@ package backtype.storm.utils; -import backtype.storm.multilang.Emission; +import backtype.storm.Config; import backtype.storm.multilang.ISerializer; -import backtype.storm.multilang.Immission; +import backtype.storm.multilang.BoltMsg; import backtype.storm.multilang.NoOutputException; +import backtype.storm.multilang.ShellMsg; import backtype.storm.multilang.SpoutMsg; import backtype.storm.task.TopologyContext; import java.io.File; @@ -18,85 +19,109 @@ public class ShellProcess implements Serializable { public static Logger LOG = Logger.getLogger(ShellProcess.class); - private Process _subprocess; - private InputStream processErrorStream; - private String[] command; - public ISerializer serializer; + private Process _subprocess; + private InputStream processErrorStream; + private String[] command; + public ISerializer serializer; - public ShellProcess(ISerializer serializer, String[] command) { + public ShellProcess(String[] command) { this.command = command; - this.serializer = serializer; } public Number launch(Map conf, TopologyContext context) { ProcessBuilder builder = new ProcessBuilder(command); builder.directory(new File(context.getCodeDir())); + this.serializer = getSerializer(conf); + Number pid; try { - _subprocess = builder.start(); - processErrorStream = _subprocess.getErrorStream(); - serializer.initialize(_subprocess.getOutputStream(), _subprocess.getInputStream()); - pid = serializer.connect(conf, context); + _subprocess = builder.start(); + processErrorStream = _subprocess.getErrorStream(); + serializer.initialize(_subprocess.getOutputStream(), + _subprocess.getInputStream()); + pid = serializer.connect(conf, context); } catch (IOException e) { - throw new RuntimeException("Error when launching multilang subprocess\n" + getErrorsString(), e); + throw new RuntimeException( + "Error when launching multilang subprocess\n" + + getErrorsString(), e); } catch (NoOutputException e) { throw new RuntimeException(e + getErrorsString() + "\n"); - } + } return pid; } + private ISerializer getSerializer(Map conf) { + //get factory class name + String serializer_className = (String)conf.get(Config.STORM_MULTILANG_SERIALIZER); + LOG.info("Storm multilang serializer:" + serializer_className); + + ISerializer serializer = null; + try { + //create a factory class + Class klass = Class.forName(serializer_className); + //obtain a serializer object + Object obj = klass.newInstance(); + serializer = (ISerializer)obj; + } catch(Exception e) { + throw new RuntimeException("Failed to construct multilang serializer from serializer " + serializer_className, e); + } + return serializer; + } + public void destroy() { _subprocess.destroy(); } - public Emission readEmission() throws IOException { - try { - return serializer.readEmission(); - } catch (NoOutputException e) { + public ShellMsg readShellMsg() throws IOException { + try { + return serializer.readShellMsg(); + } catch (NoOutputException e) { throw new RuntimeException(e + getErrorsString() + "\n"); - } + } } - public void writeImmission(Immission msg) throws IOException { - serializer.writeImmission(msg); - // drain the error stream to avoid dead lock because of full error stream buffer + public void writeBoltMsg(BoltMsg msg) throws IOException { + serializer.writeBoltMsg(msg); + // drain the error stream to avoid dead lock because of full error + // stream buffer drainErrorStream(); } public void writeSpoutMsg(SpoutMsg msg) throws IOException { - serializer.writeSpoutMsg(msg); - // drain the error stream to avoid dead lock because of full error stream buffer + serializer.writeSpoutMsg(msg); + // drain the error stream to avoid dead lock because of full error + // stream buffer drainErrorStream(); } public void writeTaskIds(List taskIds) throws IOException { - serializer.writeTaskIds(taskIds); - // drain the error stream to avoid dead lock because of full error stream buffer + serializer.writeTaskIds(taskIds); + // drain the error stream to avoid dead lock because of full error + // stream buffer drainErrorStream(); } - public void drainErrorStream() - { + public void drainErrorStream() { try { - while (processErrorStream.available() > 0) - { + while (processErrorStream.available() > 0) { int bufferSize = processErrorStream.available(); - byte[] errorReadingBuffer = new byte[bufferSize]; + byte[] errorReadingBuffer = new byte[bufferSize]; processErrorStream.read(errorReadingBuffer, 0, bufferSize); - LOG.info("Got error from shell process: " + new String(errorReadingBuffer)); + LOG.info("Got error from shell process: " + + new String(errorReadingBuffer)); } - } catch(Exception e) { + } catch (Exception e) { } } public String getErrorsString() { - if(processErrorStream!=null) { + if (processErrorStream != null) { try { return IOUtils.toString(processErrorStream); - } catch(IOException e) { + } catch (IOException e) { return "(Unable to capture error stream)"; } } else { From 52948f56511a57cd273431b794819ddc9a343bee Mon Sep 17 00:00:00 2001 From: John Gilmore Date: Wed, 9 Oct 2013 11:51:20 +0200 Subject: [PATCH 224/349] Added newline to NoOutputException --- .../src/jvm/backtype/storm/multilang/NoOutputException.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java b/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java index fd5ef72639..768f65fe01 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java +++ b/storm-core/src/jvm/backtype/storm/multilang/NoOutputException.java @@ -20,4 +20,4 @@ public NoOutputException(String message, Throwable cause) { public NoOutputException(Throwable cause) { super(cause); } -} \ No newline at end of file +} From 63afb94cd0d8aa769898ba80ed2b020a6379b52f Mon Sep 17 00:00:00 2001 From: John Gilmore Date: Wed, 9 Oct 2013 15:29:21 +0200 Subject: [PATCH 225/349] Re-added need_task_ids option --- .../src/jvm/backtype/storm/multilang/JsonSerializer.java | 9 +++++++++ .../src/jvm/backtype/storm/multilang/ShellMsg.java | 9 +++++++++ storm-core/src/jvm/backtype/storm/spout/ShellSpout.java | 4 +++- storm-core/src/jvm/backtype/storm/task/ShellBolt.java | 4 +++- 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java b/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java index ceb8cb3ed4..8afa4bbd59 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java +++ b/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java @@ -89,6 +89,15 @@ public ShellMsg readShellMsg() throws IOException, NoOutputException { Object taskObj = msg.get("task"); if (taskObj != null) { shellMsg.setTask((Long) taskObj); + } else { + shellMsg.setTask(0); + } + + Object need_task_ids = msg.get("need_task_ids"); + if (need_task_ids == null || ((Boolean) need_task_ids).booleanValue()) { + shellMsg.setNeedTaskIds(true); + } else { + shellMsg.setNeedTaskIds(false); } shellMsg.setTuple((List) msg.get("tuple")); diff --git a/storm-core/src/jvm/backtype/storm/multilang/ShellMsg.java b/storm-core/src/jvm/backtype/storm/multilang/ShellMsg.java index d56ba7854c..4381c07db5 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/ShellMsg.java +++ b/storm-core/src/jvm/backtype/storm/multilang/ShellMsg.java @@ -23,6 +23,7 @@ public class ShellMsg { private long task; private String msg; private List tuple; + private boolean needTaskIds; public String getCommand() { return command; @@ -93,4 +94,12 @@ public void addTuple(Object tuple) { } this.tuple.add(tuple); } + + public boolean areTaskIdsNeeded() { + return needTaskIds; + } + + public void setNeedTaskIds(boolean needTaskIds) { + this.needTaskIds = needTaskIds; + } } diff --git a/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java b/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java index 72cebe28c2..bee56fe52b 100644 --- a/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java +++ b/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java @@ -87,7 +87,9 @@ private void querySubprocess() { Object messageId = shellMsg.getId(); if (task == 0) { List outtasks = _collector.emit(stream, tuple, messageId); - _process.writeTaskIds(outtasks); + if (shellMsg.areTaskIdsNeeded()) { + _process.writeTaskIds(outtasks); + } } else { _collector.emitDirect((int) task.longValue(), stream, tuple, messageId); diff --git a/storm-core/src/jvm/backtype/storm/task/ShellBolt.java b/storm-core/src/jvm/backtype/storm/task/ShellBolt.java index 6a6c30520c..fb07986974 100644 --- a/storm-core/src/jvm/backtype/storm/task/ShellBolt.java +++ b/storm-core/src/jvm/backtype/storm/task/ShellBolt.java @@ -190,7 +190,9 @@ private void handleEmit(ShellMsg shellMsg) throws InterruptedException { if(shellMsg.getTask() == 0) { List outtasks = _collector.emit(shellMsg.getStream(), anchors, shellMsg.getTuple()); - _pendingWrites.put(outtasks); + if (shellMsg.areTaskIdsNeeded()) { + _pendingWrites.put(outtasks); + } } else { _collector.emitDirect((int) shellMsg.getTask(), shellMsg.getStream(), anchors, shellMsg.getTuple()); From 32b371857ddec5669f65fa779f6a0b098c1cb0a1 Mon Sep 17 00:00:00 2001 From: John Gilmore Date: Wed, 9 Oct 2013 16:40:05 +0200 Subject: [PATCH 226/349] Added json msg and id parsing. --- .../src/jvm/backtype/storm/multilang/JsonSerializer.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java b/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java index 8afa4bbd59..70d8e5b79b 100644 --- a/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java +++ b/storm-core/src/jvm/backtype/storm/multilang/JsonSerializer.java @@ -81,6 +81,12 @@ public ShellMsg readShellMsg() throws IOException, NoOutputException { String command = (String) msg.get("command"); shellMsg.setCommand(command); + String id = (String) msg.get("id"); + shellMsg.setId(id); + + String log = (String) msg.get("msg"); + shellMsg.setMsg(log); + String stream = (String) msg.get("stream"); if (stream == null) stream = Utils.DEFAULT_STREAM_ID; @@ -92,7 +98,7 @@ public ShellMsg readShellMsg() throws IOException, NoOutputException { } else { shellMsg.setTask(0); } - + Object need_task_ids = msg.get("need_task_ids"); if (need_task_ids == null || ((Boolean) need_task_ids).booleanValue()) { shellMsg.setNeedTaskIds(true); From 5699373be3842310171aa9b51059d94c03dca82a Mon Sep 17 00:00:00 2001 From: John Gilmore Date: Wed, 9 Oct 2013 17:00:10 +0200 Subject: [PATCH 227/349] Allow for null boltMsg when polling --- storm-core/src/jvm/backtype/storm/task/ShellBolt.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storm-core/src/jvm/backtype/storm/task/ShellBolt.java b/storm-core/src/jvm/backtype/storm/task/ShellBolt.java index fb07986974..1649fbed23 100644 --- a/storm-core/src/jvm/backtype/storm/task/ShellBolt.java +++ b/storm-core/src/jvm/backtype/storm/task/ShellBolt.java @@ -116,8 +116,8 @@ public void run() { _process.writeBoltMsg((BoltMsg)write); } else if (write instanceof List) { _process.writeTaskIds((List)write); - } else { - throw new RuntimeException("Cannot write object to bolt:\n" + write.toString()); + } else if (write != null) { + throw new RuntimeException("Unknown class type to write: " + write.getClass().getName()); } } catch (InterruptedException e) { } catch (Throwable t) { From 16ba0c7fbba5b49ed4054f7aba7fc7e15e144c77 Mon Sep 17 00:00:00 2001 From: John Gilmore Date: Thu, 10 Oct 2013 14:28:08 +0200 Subject: [PATCH 228/349] Removed pure whitespace changes of project.clj and bootstrap.clj --- project.clj | 2 +- storm-core/src/clj/backtype/storm/bootstrap.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 6e59efca6e..ed1fdbc3f1 100644 --- a/project.clj +++ b/project.clj @@ -10,7 +10,7 @@ :archive "https://groups.google.com/group/storm-user" :post "storm-user@googlegroups.com"} :dependencies [~@DEPENDENCIES] - :plugins [[~'lein-sub "0.2.1"]] + :plugins [[~'lein-sub "0.2.1"]] :min-lein-version "2.0.0" :sub [~@MODULES] )) diff --git a/storm-core/src/clj/backtype/storm/bootstrap.clj b/storm-core/src/clj/backtype/storm/bootstrap.clj index 9edbe8e54c..70b80057ea 100644 --- a/storm-core/src/clj/backtype/storm/bootstrap.clj +++ b/storm-core/src/clj/backtype/storm/bootstrap.clj @@ -17,7 +17,7 @@ (import (quote [backtype.storm.task IBolt IOutputCollector OutputCollector TopologyContext ShellBolt GeneralTopologyContext WorkerTopologyContext])) - (import (quote [backtype.storm.coordination CoordinatedBolt CoordinatedBolt$SourceArgs + (import (quote [backtype.storm.coordination CoordinatedBolt CoordinatedBolt$SourceArgs IBatchBolt BatchBoltExecutor])) (import (quote [backtype.storm.drpc KeyedFairBolt])) (import (quote [backtype.storm.daemon Shutdownable])) From dfbd045966a9a909aaa0e7e31eac59639a86c3c9 Mon Sep 17 00:00:00 2001 From: John Gilmore Date: Thu, 10 Oct 2013 14:58:51 +0200 Subject: [PATCH 229/349] Added some trailing whitespace to Config.java --- storm-core/src/jvm/backtype/storm/Config.java | 122 +++++++++--------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java index f94f3765d6..f8f687cf19 100644 --- a/storm-core/src/jvm/backtype/storm/Config.java +++ b/storm-core/src/jvm/backtype/storm/Config.java @@ -10,11 +10,11 @@ import java.util.Map; /** - * Topology configs are specified as a plain old map. This class provides a - * convenient way to create a topology config map by providing setter methods for - * all the configs that can be set. It also makes it easier to do things like add + * Topology configs are specified as a plain old map. This class provides a + * convenient way to create a topology config map by providing setter methods for + * all the configs that can be set. It also makes it easier to do things like add * serializations. - * + * *

This class also provides constants for all the configurations possible on * a Storm cluster and Storm topology. Each constant is paired with a schema * that defines the validity criterion of the corresponding field. Default @@ -22,7 +22,7 @@ * *

Note that you may put other configurations in any of the configs. Storm * will ignore anything it doesn't recognize, but your topologies are free to make - * use of them by reading them in the prepare method of Bolts or the open method of + * use of them by reading them in the prepare method of Bolts or the open method of * Spouts.

*/ public class Config extends HashMap { @@ -42,25 +42,25 @@ public class Config extends HashMap { /** * Netty based messaging: The buffer size for send/recv buffer */ - public static final String STORM_MESSAGING_NETTY_BUFFER_SIZE = "storm.messaging.netty.buffer_size"; + public static final String STORM_MESSAGING_NETTY_BUFFER_SIZE = "storm.messaging.netty.buffer_size"; public static final Object STORM_MESSAGING_NETTY_BUFFER_SIZE_SCHEMA = Number.class; /** * Netty based messaging: The max # of retries that a peer will perform when a remote is not accessible */ - public static final String STORM_MESSAGING_NETTY_MAX_RETRIES = "storm.messaging.netty.max_retries"; + public static final String STORM_MESSAGING_NETTY_MAX_RETRIES = "storm.messaging.netty.max_retries"; public static final Object STORM_MESSAGING_NETTY_MAX_RETRIES_SCHEMA = Number.class; /** - * Netty based messaging: The min # of milliseconds that a peer will wait. + * Netty based messaging: The min # of milliseconds that a peer will wait. */ - public static final String STORM_MESSAGING_NETTY_MIN_SLEEP_MS = "storm.messaging.netty.min_wait_ms"; + public static final String STORM_MESSAGING_NETTY_MIN_SLEEP_MS = "storm.messaging.netty.min_wait_ms"; public static final Object STORM_MESSAGING_NETTY_MIN_SLEEP_MS_SCHEMA = Number.class; /** - * Netty based messaging: The max # of milliseconds that a peer will wait. + * Netty based messaging: The max # of milliseconds that a peer will wait. */ - public static final String STORM_MESSAGING_NETTY_MAX_SLEEP_MS = "storm.messaging.netty.max_wait_ms"; + public static final String STORM_MESSAGING_NETTY_MAX_SLEEP_MS = "storm.messaging.netty.max_wait_ms"; public static final Object STORM_MESSAGING_NETTY_MAX_SLEEP_MS_SCHEMA = Number.class; /** @@ -97,7 +97,7 @@ public class Config extends HashMap { /** * A global task scheduler used to assign topologies's tasks to supervisors' wokers. - * + * * If this is not set, a default system scheduler will be used. */ public static final String STORM_SCHEDULER = "storm.scheduler"; @@ -110,9 +110,9 @@ public class Config extends HashMap { public static final Object STORM_CLUSTER_MODE_SCHEMA = String.class; /** - * The hostname the supervisors/workers should report to nimbus. If unset, Storm will + * The hostname the supervisors/workers should report to nimbus. If unset, Storm will * get the hostname to report by calling InetAddress.getLocalHost().getCanonicalHostName(). - * + * * You should set this config when you dont have a DNS which supervisors/workers * can utilize to find each other based on hostname got from calls to * InetAddress.getLocalHost().getCanonicalHostName(). @@ -127,16 +127,16 @@ public class Config extends HashMap { public static final Object STORM_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class; /** - * The serializer class for ListDelegate (tuple payload). + * The serializer class for ListDelegate (tuple payload). * The default serializer will be ListDelegateSerializer */ public static final String TOPOLOGY_TUPLE_SERIALIZER = "topology.tuple.serializer"; public static final Object TOPOLOGY_TUPLE_SERIALIZER_SCHEMA = String.class; /** - * Whether or not to use ZeroMQ for messaging in local mode. If this is set - * to false, then Storm will use a pure-Java messaging system. The purpose - * of this flag is to make it easy to run Storm in local mode by eliminating + * Whether or not to use ZeroMQ for messaging in local mode. If this is set + * to false, then Storm will use a pure-Java messaging system. The purpose + * of this flag is to make it easy to run Storm in local mode by eliminating * the need for native dependencies, which can be difficult to install. * * Defaults to false. @@ -274,7 +274,7 @@ public class Config extends HashMap { public static final Object NIMBUS_TASK_LAUNCH_SECS_SCHEMA = Number.class; /** - * Whether or not nimbus should reassign tasks if it detects that a task goes down. + * Whether or not nimbus should reassign tasks if it detects that a task goes down. * Defaults to true, and it's not recommended to change this value. */ public static final String NIMBUS_REASSIGN = "nimbus.reassign"; @@ -338,19 +338,19 @@ public class Config extends HashMap { public static final Object DRPC_PORT_SCHEMA = Number.class; /** - * DRPC thrift server worker threads + * DRPC thrift server worker threads */ public static final String DRPC_WORKER_THREADS = "drpc.worker.threads"; public static final Object DRPC_WORKER_THREADS_SCHEMA = Number.class; /** - * DRPC thrift server queue size + * DRPC thrift server queue size */ public static final String DRPC_QUEUE_SIZE = "drpc.queue.size"; public static final Object DRPC_QUEUE_SIZE_SCHEMA = Number.class; /** - * This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back. + * This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back. */ public static final String DRPC_INVOCATIONS_PORT = "drpc.invocations.port"; public static final Object DRPC_INVOCATIONS_PORT_SCHEMA = Number.class; @@ -498,8 +498,8 @@ public class Config extends HashMap { /** * How many instances to create for a spout/bolt. A task runs on a thread with zero or more * other tasks for the same spout/bolt. The number of tasks for a spout/bolt is always - * the same throughout the lifetime of a topology, but the number of executors (threads) for - * a spout/bolt can change over time. This allows a topology to scale to more or less resources + * the same throughout the lifetime of a topology, but the number of executors (threads) for + * a spout/bolt can change over time. This allows a topology to scale to more or less resources * without redeploying the topology or violating the constraints of Storm (such as a fields grouping * guaranteeing that the same value goes to the same task). */ @@ -538,8 +538,8 @@ public class Config extends HashMap { /** * A list of classes that customize storm's kryo instance during start-up. - * Each listed class name must implement IKryoDecorator. During start-up the - * listed class is instantiated with 0 arguments, then its 'decorate' method + * Each listed class name must implement IKryoDecorator. During start-up the + * listed class is instantiated with 0 arguments, then its 'decorate' method * is called with storm's kryo instance as the only argument. */ public static final String TOPOLOGY_KRYO_DECORATORS = "topology.kryo.decorators"; @@ -569,7 +569,7 @@ public class Config extends HashMap { /* * A list of classes implementing IMetricsConsumer (See storm.yaml.example for exact config format). - * Each listed class will be routed all the metrics data generated by the storm metrics API. + * Each listed class will be routed all the metrics data generated by the storm metrics API. * Each listed class maps 1:1 to a system bolt named __metrics_ClassName#N, and it's parallelism is configurable. */ public static final String TOPOLOGY_METRICS_CONSUMER_REGISTER = "topology.metrics.consumer.register"; @@ -585,24 +585,24 @@ public class Config extends HashMap { /** - * The maximum number of tuples that can be pending on a spout task at any given time. - * This config applies to individual tasks, not to spouts or topologies as a whole. - * + * The maximum number of tuples that can be pending on a spout task at any given time. + * This config applies to individual tasks, not to spouts or topologies as a whole. + * * A pending tuple is one that has been emitted from a spout but has not been acked or failed yet. - * Note that this config parameter has no effect for unreliable spouts that don't tag + * Note that this config parameter has no effect for unreliable spouts that don't tag * their tuples with a message id. */ - public static final String TOPOLOGY_MAX_SPOUT_PENDING="topology.max.spout.pending"; + public static final String TOPOLOGY_MAX_SPOUT_PENDING="topology.max.spout.pending"; public static final Object TOPOLOGY_MAX_SPOUT_PENDING_SCHEMA = Number.class; /** * A class that implements a strategy for what to do when a spout needs to wait. Waiting is * triggered in one of two conditions: - * + * * 1. nextTuple emits no tuples * 2. The spout has hit maxSpoutPending and can't emit any more tuples */ - public static final String TOPOLOGY_SPOUT_WAIT_STRATEGY="topology.spout.wait.strategy"; + public static final String TOPOLOGY_SPOUT_WAIT_STRATEGY="topology.spout.wait.strategy"; public static final Object TOPOLOGY_SPOUT_WAIT_STRATEGY_SCHEMA = String.class; /** @@ -625,7 +625,7 @@ public class Config extends HashMap { public static final Object TOPOLOGY_STATS_SAMPLE_RATE_SCHEMA = Number.class; /** - * The time period that builtin metrics data in bucketed into. + * The time period that builtin metrics data in bucketed into. */ public static final String TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS="topology.builtin.metrics.bucket.size.secs"; public static final Object TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS_SCHEMA = Number.class; @@ -652,7 +652,7 @@ public class Config extends HashMap { /** * A list of task hooks that are automatically added to every spout and bolt in the topology. An example - * of when you'd do this is to add a hook that integrates with your internal + * of when you'd do this is to add a hook that integrates with your internal * monitoring system. These hooks are instantiated using the zero-arg constructor. */ public static final String TOPOLOGY_AUTO_TASK_HOOKS="topology.auto.task.hooks"; @@ -666,7 +666,7 @@ public class Config extends HashMap { public static final Object TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE_SCHEMA = ConfigValidation.PowerOf2Validator; /** - * The maximum number of messages to batch from the thread receiving off the network to the + * The maximum number of messages to batch from the thread receiving off the network to the * executor queues. Must be a power of 2. */ public static final String TOPOLOGY_RECEIVER_BUFFER_SIZE="topology.receiver.buffer.size"; @@ -700,14 +700,14 @@ public class Config extends HashMap { public static final Object TOPOLOGY_DISRUPTOR_WAIT_STRATEGY_SCHEMA = String.class; /** - * The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed + * The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed * via the TopologyContext. */ public static final String TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE="topology.worker.shared.thread.pool.size"; public static final Object TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE_SCHEMA = Number.class; /** - * The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example, + * The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example, * an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be * reported to Zookeeper per task for every 10 second interval of time. */ @@ -776,9 +776,9 @@ public class Config extends HashMap { /** * This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers) - * for the java.library.path value. java.library.path tells the JVM where + * for the java.library.path value. java.library.path tells the JVM where * to look for native libraries. It is necessary to set this config correctly since - * Storm uses the ZeroMQ and JZMQ native libs. + * Storm uses the ZeroMQ and JZMQ native libs. */ public static final String JAVA_LIBRARY_PATH = "java.library.path"; public static final Object JAVA_LIBRARY_PATH_SCHEMA = String.class; @@ -800,17 +800,17 @@ public class Config extends HashMap { public static void setDebug(Map conf, boolean isOn) { conf.put(Config.TOPOLOGY_DEBUG, isOn); - } + } public void setDebug(boolean isOn) { setDebug(this, isOn); } - + @Deprecated public void setOptimize(boolean isOn) { put(Config.TOPOLOGY_OPTIMIZE, isOn); - } - + } + public static void setNumWorkers(Map conf, int workers) { conf.put(Config.TOPOLOGY_WORKERS, workers); } @@ -826,7 +826,7 @@ public static void setNumAckers(Map conf, int numExecutors) { public void setNumAckers(int numExecutors) { setNumAckers(this, numExecutors); } - + public static void setMessageTimeoutSecs(Map conf, int secs) { conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, secs); } @@ -834,7 +834,7 @@ public static void setMessageTimeoutSecs(Map conf, int secs) { public void setMessageTimeoutSecs(int secs) { setMessageTimeoutSecs(this, secs); } - + public static void registerSerialization(Map conf, Class klass) { getRegisteredSerializations(conf).add(klass.getName()); } @@ -842,17 +842,17 @@ public static void registerSerialization(Map conf, Class klass) { public void registerSerialization(Class klass) { registerSerialization(this, klass); } - + public static void registerSerialization(Map conf, Class klass, Class serializerClass) { Map register = new HashMap(); register.put(klass.getName(), serializerClass.getName()); - getRegisteredSerializations(conf).add(register); + getRegisteredSerializations(conf).add(register); } public void registerSerialization(Class klass, Class serializerClass) { registerSerialization(this, klass, serializerClass); } - + public void registerMetricsConsumer(Class klass, Object argument, long parallelismHint) { HashMap m = new HashMap(); m.put("class", klass.getCanonicalName()); @@ -880,7 +880,7 @@ public static void registerDecorator(Map conf, Class k public void registerDecorator(Class klass) { registerDecorator(this, klass); } - + public static void setKryoFactory(Map conf, Class klass) { conf.put(Config.TOPOLOGY_KRYO_FACTORY, klass.getName()); } @@ -896,7 +896,7 @@ public static void setSkipMissingKryoRegistrations(Map conf, boolean skip) { public void setSkipMissingKryoRegistrations(boolean skip) { setSkipMissingKryoRegistrations(this, skip); } - + public static void setMaxTaskParallelism(Map conf, int max) { conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, max); } @@ -904,7 +904,7 @@ public static void setMaxTaskParallelism(Map conf, int max) { public void setMaxTaskParallelism(int max) { setMaxTaskParallelism(this, max); } - + public static void setMaxSpoutPending(Map conf, int max) { conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, max); } @@ -912,23 +912,23 @@ public static void setMaxSpoutPending(Map conf, int max) { public void setMaxSpoutPending(int max) { setMaxSpoutPending(this, max); } - + public static void setStatsSampleRate(Map conf, double rate) { conf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, rate); - } + } public void setStatsSampleRate(double rate) { setStatsSampleRate(this, rate); - } + } public static void setFallBackOnJavaSerialization(Map conf, boolean fallback) { conf.put(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION, fallback); - } + } public void setFallBackOnJavaSerialization(boolean fallback) { setFallBackOnJavaSerialization(this, fallback); - } - + } + private static List getRegisteredSerializations(Map conf) { List ret; if(!conf.containsKey(Config.TOPOLOGY_KRYO_REGISTER)) { @@ -939,13 +939,13 @@ private static List getRegisteredSerializations(Map conf) { conf.put(Config.TOPOLOGY_KRYO_REGISTER, ret); return ret; } - + private static List getRegisteredDecorators(Map conf) { List ret; if(!conf.containsKey(Config.TOPOLOGY_KRYO_DECORATORS)) { ret = new ArrayList(); } else { - ret = new ArrayList((List) conf.get(Config.TOPOLOGY_KRYO_DECORATORS)); + ret = new ArrayList((List) conf.get(Config.TOPOLOGY_KRYO_DECORATORS)); } conf.put(Config.TOPOLOGY_KRYO_DECORATORS, ret); return ret; From 5ba093862560aa3ca2cff002e2e38e1e37753670 Mon Sep 17 00:00:00 2001 From: John Gilmore Date: Mon, 18 Nov 2013 16:31:44 +0200 Subject: [PATCH 230/349] Added debugging info to ShellSpout and pulled in latest upstream master. --- storm-core/src/jvm/backtype/storm/spout/ShellSpout.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java b/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java index bee56fe52b..6840938141 100644 --- a/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java +++ b/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java @@ -94,6 +94,8 @@ private void querySubprocess() { _collector.emitDirect((int) task.longValue(), stream, tuple, messageId); } + } else { + throw new RuntimeException("Unknown command received: " + command); } } } catch (IOException e) { From 8cde5508b5e4828982aa4f1ff2e70ba6fd10f4b9 Mon Sep 17 00:00:00 2001 From: John Gilmore Date: Fri, 29 Nov 2013 14:47:31 +0200 Subject: [PATCH 231/349] Added null check for anchors. Log a component's error stream under it name. --- .../jvm/backtype/storm/spout/ShellSpout.java | 8 +++--- .../jvm/backtype/storm/task/ShellBolt.java | 19 +++++++------ .../backtype/storm/utils/ShellProcess.java | 28 ++++++++----------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java b/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java index 6840938141..bd5f84e736 100644 --- a/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java +++ b/storm-core/src/jvm/backtype/storm/spout/ShellSpout.java @@ -31,6 +31,7 @@ public ShellSpout(String... command) { public void open(Map stormConf, TopologyContext context, SpoutOutputCollector collector) { _collector = collector; + _process = new ShellProcess(_command); Number subpid = _process.launch(stormConf, context); @@ -51,7 +52,7 @@ public void nextTuple() { } public void ack(Object msgId) { - if (spoutMsg == null) { + if (spoutMsg == null) { spoutMsg = new SpoutMsg(); } spoutMsg.setCommand("ack"); @@ -60,7 +61,7 @@ public void ack(Object msgId) { } public void fail(Object msgId) { - if (spoutMsg == null) { + if (spoutMsg == null) { spoutMsg = new SpoutMsg(); } spoutMsg.setCommand("fail"); @@ -91,8 +92,7 @@ private void querySubprocess() { _process.writeTaskIds(outtasks); } } else { - _collector.emitDirect((int) task.longValue(), stream, - tuple, messageId); + _collector.emitDirect((int) task.longValue(), stream, tuple, messageId); } } else { throw new RuntimeException("Unknown command received: " + command); diff --git a/storm-core/src/jvm/backtype/storm/task/ShellBolt.java b/storm-core/src/jvm/backtype/storm/task/ShellBolt.java index 8efd904dd5..0016d66334 100644 --- a/storm-core/src/jvm/backtype/storm/task/ShellBolt.java +++ b/storm-core/src/jvm/backtype/storm/task/ShellBolt.java @@ -184,14 +184,17 @@ private void handleError(String msg) { } private void handleEmit(ShellMsg shellMsg) throws InterruptedException { - List anchors = new ArrayList(); - for (String anchor : shellMsg.getAnchors()) { - Tuple t = _inputs.get(anchor); - if (t == null) { - throw new RuntimeException("Anchored onto " + anchor + " after ack/fail"); - } - anchors.add(t); - } + List anchors = new ArrayList(); + List recvAnchors = shellMsg.getAnchors(); + if (recvAnchors != null) { + for (String anchor : recvAnchors) { + Tuple t = _inputs.get(anchor); + if (t == null) { + throw new RuntimeException("Anchored onto " + anchor + " after ack/fail"); + } + anchors.add(t); + } + } if(shellMsg.getTask() == 0) { List outtasks = _collector.emit(shellMsg.getStream(), anchors, shellMsg.getTuple()); diff --git a/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java b/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java index 8871a77b9f..2c3b3a3fc6 100644 --- a/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java +++ b/storm-core/src/jvm/backtype/storm/utils/ShellProcess.java @@ -19,6 +19,7 @@ public class ShellProcess implements Serializable { public static Logger LOG = Logger.getLogger(ShellProcess.class); + public static Logger ShellLogger; private Process _subprocess; private InputStream processErrorStream; private String[] command; @@ -32,14 +33,15 @@ public Number launch(Map conf, TopologyContext context) { ProcessBuilder builder = new ProcessBuilder(command); builder.directory(new File(context.getCodeDir())); + ShellLogger = Logger.getLogger(context.getThisComponentId()); + this.serializer = getSerializer(conf); Number pid; try { _subprocess = builder.start(); processErrorStream = _subprocess.getErrorStream(); - serializer.initialize(_subprocess.getOutputStream(), - _subprocess.getInputStream()); + serializer.initialize(_subprocess.getOutputStream(), _subprocess.getInputStream()); pid = serializer.connect(conf, context); } catch (IOException e) { throw new RuntimeException( @@ -83,35 +85,29 @@ public ShellMsg readShellMsg() throws IOException { public void writeBoltMsg(BoltMsg msg) throws IOException { serializer.writeBoltMsg(msg); - // drain the error stream to avoid dead lock because of full error - // stream buffer - drainErrorStream(); + // Log any info sent on the error stream + logErrorStream(); } public void writeSpoutMsg(SpoutMsg msg) throws IOException { serializer.writeSpoutMsg(msg); - // drain the error stream to avoid dead lock because of full error - // stream buffer - drainErrorStream(); + // Log any info sent on the error stream + logErrorStream(); } public void writeTaskIds(List taskIds) throws IOException { serializer.writeTaskIds(taskIds); - // drain the error stream to avoid dead lock because of full error - // stream buffer - drainErrorStream(); + // Log any info sent on the error stream + logErrorStream(); } - public void drainErrorStream() { + public void logErrorStream() { try { while (processErrorStream.available() > 0) { int bufferSize = processErrorStream.available(); byte[] errorReadingBuffer = new byte[bufferSize]; - processErrorStream.read(errorReadingBuffer, 0, bufferSize); - - LOG.info("Got error from shell process: " - + new String(errorReadingBuffer)); + ShellLogger.info(new String(errorReadingBuffer)); } } catch (Exception e) { } From 8012cd7a4a8665469ccd4e89c4b7013f52007021 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Sat, 7 Dec 2013 00:24:28 -0500 Subject: [PATCH 232/349] align with version 0.9.0.1, clean up m2-pom.xml and align with project.clj, update README --- README.markdown | 5 ++++- m2-pom.xml | 49 ++++++++++++++++++++++--------------------------- project.clj | 2 +- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/README.markdown b/README.markdown index 963e9c2509..69941f2f03 100644 --- a/README.markdown +++ b/README.markdown @@ -88,8 +88,11 @@ the [Maven installation instructions](http://maven.apache.org/download.cgi). storm-starter contains [m2-pom.xml](m2-pom.xml) which can be used with Maven using the `-f` option. For example, to compile and run `WordCountTopology` in local mode, use the command: - $ mvn -f m2-pom.xml compile exec:java -Dexec.classpathScope=compile -Dexec.mainClass=storm.starter.WordCountTopology + $ mvn -f m2-pom.xml compile exec:java -Dstorm.topology=storm.starter.WordCountTopology +You can also run clojure topologies with Maven: + + $ mvn -f m2-pom.xml compile exec:java -Dstorm.topology=storm.starter.clj.word_count ## Packaging storm-starter for use on a Storm cluster diff --git a/m2-pom.xml b/m2-pom.xml index 24293bca1a..8d04fc9d0e 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -22,10 +22,6 @@ clojars.org http://clojars.org/repo - - twitter4j - http://twitter4j.org/maven2 - @@ -38,7 +34,7 @@ org.testng testng - 6.8 + 6.8.5 test @@ -62,30 +58,10 @@ storm storm - 0.8.2 + 0.9.0.1 provided - - org.twitter4j - twitter4j-core - 2.2.6-SNAPSHOT - - - org.twitter4j - twitter4j-stream - 2.2.6-SNAPSHOT - - - com.googlecode.json-simple - json-simple - 1.1 - - - com.google.guava - guava - 13.0.1 - commons-collections commons-collections @@ -134,7 +110,7 @@ com.theoryinpractise clojure-maven-plugin - 1.3.8 + 1.3.12 true @@ -159,6 +135,25 @@ + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + + + exec + + + + + java + true + false + compile + ${storm.topology} + + org.apache.maven.plugins maven-compiler-plugin diff --git a/project.clj b/project.clj index 89192751e8..ae6c3778d1 100644 --- a/project.clj +++ b/project.clj @@ -15,7 +15,7 @@ ] :profiles {:dev - {:dependencies [[storm "0.8.2"] + {:dependencies [[storm "0.9.0.1"] [org.clojure/clojure "1.4.0"] [org.testng/testng "6.8.5"] [org.easytesting/fest-assert-core "2.0M8"] From 4c403845b55c38b2573455efbe519c45a0f0c045 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Sat, 7 Dec 2013 10:25:52 +0100 Subject: [PATCH 233/349] m2-pom.xml: fix indentation / whitespace --- m2-pom.xml | 317 +++++++++++++++++++++++++++-------------------------- 1 file changed, 160 insertions(+), 157 deletions(-) diff --git a/m2-pom.xml b/m2-pom.xml index 8d04fc9d0e..07b642562a 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -1,167 +1,170 @@ - 4.0.0 + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 - storm.starter - storm-starter - 0.0.1-SNAPSHOT - jar + storm.starter + storm-starter + 0.0.1-SNAPSHOT + jar - storm-starter - https://github.com/nathanmarz/storm-starter + storm-starter + https://github.com/nathanmarz/storm-starter - - UTF-8 - - - - github-releases - http://oss.sonatype.org/content/repositories/github-releases/ - - - clojars.org - http://clojars.org/repo - - + + UTF-8 + - - - junit - junit - 3.8.1 - test - - - org.testng - testng - 6.8.5 - test - - - org.mockito - mockito-all - 1.9.0 - test - - - org.easytesting - fest-assert-core - 2.0M8 - test - - - org.jmock - jmock - 2.6.0 - test - - - storm - storm - 0.9.0.1 - - provided - - - commons-collections - commons-collections - 3.2.1 - - + + + github-releases + http://oss.sonatype.org/content/repositories/github-releases/ + + + clojars.org + http://clojars.org/repo + + - - src/jvm - test/jvm - - - ${basedir}/multilang - - - - - - maven-assembly-plugin - - - jar-with-dependencies - - - - - - - - - - make-assembly - package - - single - - - + + + junit + junit + 3.8.1 + test + + + org.testng + testng + 6.8.5 + test + + + org.mockito + mockito-all + 1.9.0 + test + + + org.easytesting + fest-assert-core + 2.0M8 + test + + + org.jmock + jmock + 2.6.0 + test + + + storm + storm + 0.9.0.1 + + provided + + + commons-collections + commons-collections + 3.2.1 + + - + + src/jvm + test/jvm + + + ${basedir}/multilang + + - - com.theoryinpractise - clojure-maven-plugin - 1.3.12 - true - - - src/clj - - - - - compile - compile - - compile - - - - test - test - - test - - - + + + + maven-assembly-plugin + + + jar-with-dependencies + + + + + + + + + + make-assembly + package + + single + + + + - - - org.codehaus.mojo - exec-maven-plugin - 1.2.1 - - - - exec - - - - - java - true - false - compile - ${storm.topology} - - - - org.apache.maven.plugins - maven-compiler-plugin - - 1.6 - 1.6 - - - - + + com.theoryinpractise + clojure-maven-plugin + 1.3.12 + true + + + src/clj + + + + + compile + compile + + compile + + + + test + test + + test + + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + + + exec + + + + + java + true + false + compile + ${storm.topology} + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.6 + 1.6 + + + + + From 1bcefbbfc960353ec6713adb2f3342e4a92a31bb Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Sat, 7 Dec 2013 10:36:58 +0100 Subject: [PATCH 234/349] README: Add instructions for IntelliJ IDEA --- README.markdown | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.markdown b/README.markdown index 69941f2f03..7e1c8e04d4 100644 --- a/README.markdown +++ b/README.markdown @@ -9,6 +9,7 @@ Table of Contents * Getting started * Using storm-starter with Leiningen * Using storm-starter with Maven +* Using storm-starter with IntelliJ IDEA --- @@ -110,3 +111,25 @@ Use the following Maven command to run the unit tests that ship with storm-start yet run the included unit tests. $ mvn -f m2-pom.xml test + + + + +# Using storm-starter with IntelliJ IDEA + +## Importing storm-starter as a project in IDEA + +The following instructions will import storm-starter as a new project in IntelliJ IDEA. + +* Open _File > Import Project..._ and navigate to the top-level directory of your storm-starter clone (e.g. + `~/git/storm-starter`). +* Select _Create project from existing sources_ and click _Next_. +* You may now optionally change the suggested name of the project in IDEA (by default, IDEA suggests `storm-starter`). + Click _Next_ when you are done. +* Click _Next_ on the following screen -- IDEA will auto-detect the correct source files of storm-starter. +* Select the JDK to be used by IDEA for storm-starter, then click _Next_. + * At the time of this writing you should use JDK 6. + * It is strongly recommended to use Sun/Oracle JDK 6 rather than OpenJDK 6. +* Click _Next_ on the following screen about reviewing libraries found by IDEA. It is ok if none are listed. +* CLick _Next_ on the following screen about reviwing suggested modules -- IDEA will auto-detect the relevant settings. +* Click _Finish_. From b77057c5d0aa9e47469c86da7c01f4aba3bd5f43 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sat, 7 Dec 2013 10:53:18 +0000 Subject: [PATCH 235/349] updated to final version of kafka 0.8 --- README.md | 17 +---------------- pom.xml | 2 +- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 034bfa9e99..fbfcdadda4 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,4 @@ storm-kafka-0.8-plus ==================== -Port of storm-kafka to support kafka >= 0.8 - -## Installing kafka 0.8 into local maven repository - -Unfortunately the current beta pom is slightly broken ([KAFKA-974](https://issues.apache.org/jira/browse/KAFKA-974)) so you might want to build your own - -- checkout kafka 0.8 branch -- Check [KAFKA-939](https://issues.apache.org/jira/browse/KAFKA-939) for instructions on how to handle javadoc erros when publishing -- GPG: - - ```gpg2 --gen-key``` - - ```cp ~/.gnupg/secring.gpg ~/.sbt/gpg/secring.asc``` -- publish kafka to local ivy repository - - ```./sbt "++2.9.2" clean package publish-local``` -- publish kafka to local maven repo - - ```mvn install:install-file -Dfile=$HOME/.ivy2/local/org.apache.kafka/kafka_2.9.2/0.8.0-beta1/jars/kafka_2.9.2.jar -DpomFile=$HOME/.ivy2/local/org.apache.kafka/kafka_2.9.2/0.8.0-beta1/poms/kafka_2.9.2.pom``` - +Port of storm-kafka to support kafka >= 0.8 \ No newline at end of file diff --git a/pom.xml b/pom.xml index 0d1641c8a7..b2ef2e2879 100644 --- a/pom.xml +++ b/pom.xml @@ -132,7 +132,7 @@ org.apache.kafka kafka_2.9.2 - 0.8.0-beta1 + 0.8.0 org.apache.zookeeper From 74f01ec64eef63181e8d6b92a08295a279e76158 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Sat, 7 Dec 2013 11:54:59 +0100 Subject: [PATCH 236/349] Re-add Google Guava to maven deps config (accidentally removed in commit 8012cd7) --- m2-pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/m2-pom.xml b/m2-pom.xml index 07b642562a..77011a7e41 100644 --- a/m2-pom.xml +++ b/m2-pom.xml @@ -68,6 +68,11 @@ commons-collections 3.2.1 + + com.google.guava + guava + 15.0 + From 651ccfe07b542fd0b2c8c9a518fb6c1cd173bd9d Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Sat, 7 Dec 2013 12:26:11 +0100 Subject: [PATCH 237/349] AbstractRankerBolt: Emit defensive copy of Rankings object This addresses ConcurrentModificationException that might be thrown in the ranker bolts, when the Rankings object would be modified (by the emitting bolt) after sending it downstream. --- .../starter/bolt/AbstractRankerBolt.java | 2 +- src/jvm/storm/starter/tools/Rankable.java | 6 ++ .../tools/RankableObjectWithFields.java | 13 +++ src/jvm/storm/starter/tools/Rankings.java | 35 ++++++-- storm-starter.iml | 82 +++++++++++++++++ .../tools/RankableObjectWithFieldsTest.java | 21 +++++ .../jvm/storm/starter/tools/RankingsTest.java | 90 +++++++++++++++++++ 7 files changed, 243 insertions(+), 6 deletions(-) create mode 100644 storm-starter.iml diff --git a/src/jvm/storm/starter/bolt/AbstractRankerBolt.java b/src/jvm/storm/starter/bolt/AbstractRankerBolt.java index 6e2551c966..07ac843355 100644 --- a/src/jvm/storm/starter/bolt/AbstractRankerBolt.java +++ b/src/jvm/storm/starter/bolt/AbstractRankerBolt.java @@ -73,7 +73,7 @@ public final void execute(Tuple tuple, BasicOutputCollector collector) { abstract void updateRankingsWithTuple(Tuple tuple); private void emitRankings(BasicOutputCollector collector) { - collector.emit(new Values(rankings)); + collector.emit(new Values(rankings.copy())); getLogger().debug("Rankings: " + rankings); } diff --git a/src/jvm/storm/starter/tools/Rankable.java b/src/jvm/storm/starter/tools/Rankable.java index 36ba0860d9..254d920acb 100644 --- a/src/jvm/storm/starter/tools/Rankable.java +++ b/src/jvm/storm/starter/tools/Rankable.java @@ -6,4 +6,10 @@ public interface Rankable extends Comparable { long getCount(); + /** + * Note: We do not defensively copy the object wrapped by the Rankable. It is passed as is. + * + * @return a defensive copy + */ + Rankable copy(); } diff --git a/src/jvm/storm/starter/tools/RankableObjectWithFields.java b/src/jvm/storm/starter/tools/RankableObjectWithFields.java index 3b27d0d5dc..3079434f6a 100644 --- a/src/jvm/storm/starter/tools/RankableObjectWithFields.java +++ b/src/jvm/storm/starter/tools/RankableObjectWithFields.java @@ -115,4 +115,17 @@ public String toString() { buf.append("]"); return buf.toString(); } + + /** + * Note: We do not defensively copy the wrapped object and any accompanying fields. We do guarantee, however, + * do return a defensive (shallow) copy of the List object that is wrapping any accompanying fields. + * + * @return + */ + @Override + public Rankable copy() { + List shallowCopyOfFields = ImmutableList.copyOf(getFields()); + return new RankableObjectWithFields(getObject(), getCount(), shallowCopyOfFields); + } + } diff --git a/src/jvm/storm/starter/tools/Rankings.java b/src/jvm/storm/starter/tools/Rankings.java index 5ff471a75d..5076bf65ac 100644 --- a/src/jvm/storm/starter/tools/Rankings.java +++ b/src/jvm/storm/starter/tools/Rankings.java @@ -1,5 +1,6 @@ package storm.starter.tools; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.io.Serializable; @@ -25,6 +26,15 @@ public Rankings(int topN) { maxSize = topN; } + /** + * Copy constructor. + * @param other + */ + public Rankings(Rankings other) { + this(other.maxSize()); + updateWith(other); + } + /** * @return the maximum possible number (size) of ranked objects this instance can hold */ @@ -39,12 +49,20 @@ public int size() { return rankedItems.size(); } + /** + * The returned defensive copy is only "somewhat" defensive. We do, for instance, return a defensive copy of the + * enclosing List instance, and we do try to defensively copy any contained Rankable objects, too. However, the + * contract of {@link storm.starter.tools.Rankable#copy()} does not guarantee that any Object's embedded within + * a Rankable will be defensively copied, too. + * + * @return a somewhat defensive copy of ranked items + */ public List getRankings() { - return defensiveCopyOf(rankedItems); - } - - private List defensiveCopyOf(List list) { - return Lists.newArrayList(rankedItems); + List copy = Lists.newLinkedList(); + for (Rankable r: rankedItems) { + copy.add(r.copy()); + } + return ImmutableList.copyOf(copy); } public void updateWith(Rankings other) { @@ -111,4 +129,11 @@ public void pruneZeroCounts() { public String toString() { return rankedItems.toString(); } + + /** + * Creates a (defensive) copy of itself. + */ + public Rankings copy() { + return new Rankings(this); + } } \ No newline at end of file diff --git a/storm-starter.iml b/storm-starter.iml new file mode 100644 index 0000000000..a4e5116153 --- /dev/null +++ b/storm-starter.iml @@ -0,0 +1,82 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/jvm/storm/starter/tools/RankableObjectWithFieldsTest.java b/test/jvm/storm/starter/tools/RankableObjectWithFieldsTest.java index c8a14ccc03..82a89e6dfc 100644 --- a/test/jvm/storm/starter/tools/RankableObjectWithFieldsTest.java +++ b/test/jvm/storm/starter/tools/RankableObjectWithFieldsTest.java @@ -211,4 +211,25 @@ public void shouldCreateRankableObjectFromTuple() { assertThat(r.getFields()).isEqualTo(fields); } + + @DataProvider + public Object[][] copyData() { + return new Object[][]{ { new RankableObjectWithFields("foo", 0) }, { new RankableObjectWithFields("foo", 3, + "someOtherField") }, { new RankableObjectWithFields("foo", 0, "someField") } }; + } + + // TODO: What would be a good test to ensure that RankableObjectWithFields is at least somewhat defensively copied? + // The contract of Rankable#copy() returns a Rankable value, not a RankableObjectWithFields. + @Test(dataProvider = "copyData") + public void copyShouldReturnCopy(RankableObjectWithFields original) { + // given + + // when + Rankable copy = original.copy(); + + // then + assertThat(copy.getObject()).isEqualTo(original.getObject()); + assertThat(copy.getCount()).isEqualTo(original.getCount()); + } + } diff --git a/test/jvm/storm/starter/tools/RankingsTest.java b/test/jvm/storm/starter/tools/RankingsTest.java index 7d3b4da490..e1f82903cf 100644 --- a/test/jvm/storm/starter/tools/RankingsTest.java +++ b/test/jvm/storm/starter/tools/RankingsTest.java @@ -35,6 +35,57 @@ public void constructorWithNegativeOrZeroTopNShouldThrowIAE(int topN) { new Rankings(topN); } + @DataProvider + public Object[][] copyRankingsData() { + return new Object[][]{ { 5, Lists.newArrayList(A, B, C) }, { 2, Lists.newArrayList(A, B, C, D) }, + { 1, Lists.newArrayList() }, { 1, Lists.newArrayList(A) }, { 1, Lists.newArrayList(A, B) } }; + } + + @Test(dataProvider = "copyRankingsData") + public void copyConstructorShouldReturnCopy(int topN, List rankables) { + // given + Rankings rankings = new Rankings(topN); + for (Rankable r : rankables) { + rankings.updateWith(r); + } + + // when + Rankings copy = new Rankings(rankings); + + // then + assertThat(copy.maxSize()).isEqualTo(rankings.maxSize()); + assertThat(copy.getRankings()).isEqualTo(rankings.getRankings()); + } + + @DataProvider + public Object[][] defensiveCopyRankingsData() { + return new Object[][]{ { 5, Lists.newArrayList(A, B, C), Lists.newArrayList(D) }, { 2, Lists.newArrayList(A, B, C, + D), Lists.newArrayList(E, F) }, { 1, Lists.newArrayList(), Lists.newArrayList(A) }, { 1, Lists.newArrayList(A), + Lists.newArrayList(B) }, { 1, Lists.newArrayList(ZERO), Lists.newArrayList(B) }, { 1, Lists.newArrayList(ZERO), + Lists.newArrayList() } }; + } + + @Test(dataProvider = "defensiveCopyRankingsData") + public void copyConstructorShouldReturnDefensiveCopy(int topN, List rankables, List changes) { + // given + Rankings original = new Rankings(topN); + for (Rankable r : rankables) { + original.updateWith(r); + } + int expSize = original.size(); + List expRankings = original.getRankings(); + + // when + Rankings copy = new Rankings(original); + for (Rankable r : changes) { + copy.updateWith(r); + } + + // then + assertThat(original.size()).isEqualTo(expSize); + assertThat(original.getRankings()).isEqualTo(expRankings); + } + @DataProvider public Object[][] legalTopNData() { return new Object[][]{ { 1 }, { 2 }, { 1000 }, { 1000000 } }; @@ -258,4 +309,43 @@ public void run() { } assertThat(exceptions).isEmpty(); } + + @Test(dataProvider = "copyRankingsData") + public void copyShouldReturnCopy(int topN, List rankables) { + // given + Rankings rankings = new Rankings(topN); + for (Rankable r : rankables) { + rankings.updateWith(r); + } + + // when + Rankings copy = rankings.copy(); + + // then + assertThat(copy.maxSize()).isEqualTo(rankings.maxSize()); + assertThat(copy.getRankings()).isEqualTo(rankings.getRankings()); + } + + @Test(dataProvider = "defensiveCopyRankingsData") + public void copyShouldReturnDefensiveCopy(int topN, List rankables, List changes) { + // given + Rankings original = new Rankings(topN); + for (Rankable r : rankables) { + original.updateWith(r); + } + int expSize = original.size(); + List expRankings = original.getRankings(); + + // when + Rankings copy = original.copy(); + for (Rankable r : changes) { + copy.updateWith(r); + } + copy.pruneZeroCounts(); + + // then + assertThat(original.size()).isEqualTo(expSize); + assertThat(original.getRankings()).isEqualTo(expRankings); + } + } \ No newline at end of file From 8364b51c0a053e1b06229144d074f1af80e21927 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Sat, 7 Dec 2013 12:56:05 +0100 Subject: [PATCH 238/349] README: Fix instructions for IntelliJ IDEA --- README.markdown | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.markdown b/README.markdown index 7e1c8e04d4..e271ac1b03 100644 --- a/README.markdown +++ b/README.markdown @@ -121,15 +121,16 @@ yet run the included unit tests. The following instructions will import storm-starter as a new project in IntelliJ IDEA. +* Copy `m2-pom.xml` to `pom.xml`. This is requried so that IDEA (or Eclipse) can properly detect the maven + configuration. * Open _File > Import Project..._ and navigate to the top-level directory of your storm-starter clone (e.g. `~/git/storm-starter`). -* Select _Create project from existing sources_ and click _Next_. -* You may now optionally change the suggested name of the project in IDEA (by default, IDEA suggests `storm-starter`). - Click _Next_ when you are done. -* Click _Next_ on the following screen -- IDEA will auto-detect the correct source files of storm-starter. +* Select _Import project from external model_, select "Maven", and click _Next_. +* In the following screen, enable the checkbox _Import Maven projects automatically_. Leave all other values at their + defaults. Click _Next_. +* Click _Next_ on the following screen about selecting Maven projects to import. * Select the JDK to be used by IDEA for storm-starter, then click _Next_. * At the time of this writing you should use JDK 6. * It is strongly recommended to use Sun/Oracle JDK 6 rather than OpenJDK 6. -* Click _Next_ on the following screen about reviewing libraries found by IDEA. It is ok if none are listed. -* CLick _Next_ on the following screen about reviwing suggested modules -- IDEA will auto-detect the relevant settings. -* Click _Finish_. +* You may now optionally change the name of the project in IDEA. The default name suggested by IDEA is "storm-starter". + Click _Finish_ once you are done. From e7f7feac92b4ae7045b5d1c0c8cc75bb842cc20b Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 22 Dec 2013 12:48:49 +0000 Subject: [PATCH 239/349] updated to final version of storm 0.9.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b2ef2e2879..69d19ad65f 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,7 @@ storm storm - 0.9.0-wip21 + 0.9.0 provided From 5da87f3382f75fae4dfe13f8af06896615f80291 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 22 Dec 2013 13:06:52 +0000 Subject: [PATCH 240/349] preparing for tag --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 69d19ad65f..e1e45b6379 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ storm storm-kafka-0.8-plus jar - 0.1.0-SNAPSHOT + 0.1.0 storm-kafka-0.8-plus Storm module for kafka > 0.8 From a4c30ebbab9bc662877c8d063e6c89f5f78d7ad9 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 22 Dec 2013 13:20:21 +0000 Subject: [PATCH 241/349] added distribution management * bumped version * updated group id --- pom.xml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index e1e45b6379..d370aba336 100644 --- a/pom.xml +++ b/pom.xml @@ -2,10 +2,10 @@ 4.0.0 - storm + org.clojars.wurstmeister storm-kafka-0.8-plus jar - 0.1.0 + 0.2.0-SNAPSHOT storm-kafka-0.8-plus Storm module for kafka > 0.8 @@ -163,4 +163,11 @@ provided + + + clojars + Clojars repository + https://clojars.org/repo + + From a440ecc05f7bae6b3cea6b6c9d0450a577b42d58 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 22 Dec 2013 13:57:29 +0000 Subject: [PATCH 242/349] updated group id, prepared for tag --- pom.xml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d370aba336..430ecef9e8 100644 --- a/pom.xml +++ b/pom.xml @@ -2,12 +2,21 @@ 4.0.0 - org.clojars.wurstmeister + net.wurstmeister.storm storm-kafka-0.8-plus jar - 0.2.0-SNAPSHOT + 0.2.0 storm-kafka-0.8-plus Storm module for kafka > 0.8 + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + https://github.com/wurstmeister/storm-kafka-0.8-plus scm:git:git://github.com/wurstmeister/storm-kafka-0.8-plus.git scm:git:ssh://git@github.com/wurstmeister/storm-kafka-0.8-plus.git From d35a6eee117ec401e5fea992200167ead9f764d7 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 22 Dec 2013 14:18:19 +0000 Subject: [PATCH 243/349] updated readme --- README.md | 8 +++++++- pom.xml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fbfcdadda4..b734b5bffe 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,10 @@ storm-kafka-0.8-plus ==================== -Port of storm-kafka to support kafka >= 0.8 \ No newline at end of file +Port of storm-kafka to support kafka >= 0.8 + +##Usage +For information on how to use this library in your project see: + +[https://clojars.org/net.wurstmeister.storm/storm-kafka-0.8-plus](https://clojars.org/net.wurstmeister.storm/storm-kafka-0.8-plus) + diff --git a/pom.xml b/pom.xml index 430ecef9e8..8dfd316b36 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ net.wurstmeister.storm storm-kafka-0.8-plus jar - 0.2.0 + 0.3.0-SNAPSHOT storm-kafka-0.8-plus Storm module for kafka > 0.8 From e8f54d63094806a2a1364adb5e209ab2ca10f0f0 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 22 Dec 2013 15:58:01 +0000 Subject: [PATCH 244/349] use consistent formatting --- src/jvm/storm/kafka/DynamicBrokersReader.java | 158 +++--- .../kafka/DynamicPartitionConnections.java | 34 +- src/jvm/storm/kafka/HostPort.java | 50 +- src/jvm/storm/kafka/KafkaConfig.java | 16 +- src/jvm/storm/kafka/KafkaSpout.java | 45 +- src/jvm/storm/kafka/Partition.java | 50 +- src/jvm/storm/kafka/PartitionCoordinator.java | 1 + src/jvm/storm/kafka/PartitionManager.java | 100 ++-- src/jvm/storm/kafka/StaticCoordinator.java | 12 +- src/jvm/storm/kafka/StaticHosts.java | 17 +- .../kafka/StaticPartitionConnections.java | 8 +- src/jvm/storm/kafka/ZkCoordinator.java | 48 +- src/jvm/storm/kafka/ZkHosts.java | 22 +- src/jvm/storm/kafka/ZkState.java | 72 +-- src/jvm/storm/kafka/trident/Coordinator.java | 44 +- .../kafka/trident/DefaultCoordinator.java | 2 +- .../trident/GlobalPartitionInformation.java | 86 ++-- .../kafka/trident/IBatchCoordinator.java | 1 + .../storm/kafka/trident/IBrokerReader.java | 1 + src/jvm/storm/kafka/trident/KafkaUtils.java | 74 +-- src/jvm/storm/kafka/trident/MaxMetric.java | 8 +- .../trident/OpaqueTridentKafkaSpout.java | 14 +- .../kafka/trident/StaticBrokerReader.java | 6 +- .../TransactionalTridentKafkaSpout.java | 8 +- .../kafka/trident/TridentKafkaConfig.java | 6 +- .../kafka/trident/TridentKafkaEmitter.java | 466 +++++++++--------- .../storm/kafka/trident/ZkBrokerReader.java | 12 +- .../storm/kafka/DynamicBrokersReaderTest.java | 258 +++++----- 28 files changed, 821 insertions(+), 798 deletions(-) diff --git a/src/jvm/storm/kafka/DynamicBrokersReader.java b/src/jvm/storm/kafka/DynamicBrokersReader.java index ae155343b9..c802baf3ea 100644 --- a/src/jvm/storm/kafka/DynamicBrokersReader.java +++ b/src/jvm/storm/kafka/DynamicBrokersReader.java @@ -12,117 +12,117 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.util.*; +import java.util.List; +import java.util.Map; public class DynamicBrokersReader { - public static final Logger LOG = LoggerFactory.getLogger(DynamicBrokersReader.class); + public static final Logger LOG = LoggerFactory.getLogger(DynamicBrokersReader.class); private CuratorFramework _curator; private String _zkPath; private String _topic; - + public DynamicBrokersReader(Map conf, String zkStr, String zkPath, String topic) { - _zkPath = zkPath; - _topic = topic; - try { - _curator = CuratorFrameworkFactory.newClient( - zkStr, - Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), - 15000, - new RetryNTimes(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), - Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); - _curator.start(); - } catch (IOException ex) { - LOG.error("can't connect to zookeeper"); - } + _zkPath = zkPath; + _topic = topic; + try { + _curator = CuratorFrameworkFactory.newClient( + zkStr, + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), + 15000, + new RetryNTimes(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); + _curator.start(); + } catch (IOException ex) { + LOG.error("can't connect to zookeeper"); + } } - + /** - * Get all partitions with their current leaders + * Get all partitions with their current leaders */ public GlobalPartitionInformation getBrokerInfo() { - GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(); + GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(); try { - int numPartitionsForTopic = getNumPartitions(); - String brokerInfoPath = brokerPath(); - for (int partition = 0; partition < numPartitionsForTopic; partition++) { - int leader = getLeaderFor(partition); - String path = brokerInfoPath + "/" + leader; - try { - byte[] hostPortData = _curator.getData().forPath(path); - HostPort hp = getBrokerHost(hostPortData); - globalPartitionInformation.addPartition(partition, hp); - } catch(org.apache.zookeeper.KeeperException.NoNodeException e) { - LOG.error("Node {} does not exist ", path); - } - } - } catch(Exception e) { + int numPartitionsForTopic = getNumPartitions(); + String brokerInfoPath = brokerPath(); + for (int partition = 0; partition < numPartitionsForTopic; partition++) { + int leader = getLeaderFor(partition); + String path = brokerInfoPath + "/" + leader; + try { + byte[] hostPortData = _curator.getData().forPath(path); + HostPort hp = getBrokerHost(hostPortData); + globalPartitionInformation.addPartition(partition, hp); + } catch (org.apache.zookeeper.KeeperException.NoNodeException e) { + LOG.error("Node {} does not exist ", path); + } + } + } catch (Exception e) { throw new RuntimeException(e); } - LOG.info("Read partition info from zookeeper: " + globalPartitionInformation); + LOG.info("Read partition info from zookeeper: " + globalPartitionInformation); return globalPartitionInformation; } + private int getNumPartitions() { + try { + String topicBrokersPath = partitionPath(); + List children = _curator.getChildren().forPath(topicBrokersPath); + return children.size(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } - private int getNumPartitions() { - try { - String topicBrokersPath = partitionPath(); - List children = _curator.getChildren().forPath(topicBrokersPath); - return children.size(); - } catch(Exception e) { - throw new RuntimeException(e); - } - } - - public String partitionPath() { - return _zkPath + "/topics/" + _topic + "/partitions"; - } + public String partitionPath() { + return _zkPath + "/topics/" + _topic + "/partitions"; + } - public String brokerPath() { - return _zkPath + "/ids"; - } + public String brokerPath() { + return _zkPath + "/ids"; + } - /** - * get /brokers/topics/distributedTopic/partitions/1/state - * { "controller_epoch":4, "isr":[ 1, 0 ], "leader":1, "leader_epoch":1, "version":1 } - * @param partition - * @return - */ - private int getLeaderFor(long partition) { - try { - String topicBrokersPath = partitionPath(); - byte[] hostPortData = _curator.getData().forPath(topicBrokersPath + "/" + partition + "/state" ); - Map value = (Map) JSONValue.parse(new String(hostPortData, "UTF-8")); - Integer leader = ((Number) value.get("leader")).intValue(); - return leader; - } catch (Exception e) { - throw new RuntimeException(e); - } - } + /** + * get /brokers/topics/distributedTopic/partitions/1/state + * { "controller_epoch":4, "isr":[ 1, 0 ], "leader":1, "leader_epoch":1, "version":1 } + * + * @param partition + * @return + */ + private int getLeaderFor(long partition) { + try { + String topicBrokersPath = partitionPath(); + byte[] hostPortData = _curator.getData().forPath(topicBrokersPath + "/" + partition + "/state"); + Map value = (Map) JSONValue.parse(new String(hostPortData, "UTF-8")); + Integer leader = ((Number) value.get("leader")).intValue(); + return leader; + } catch (Exception e) { + throw new RuntimeException(e); + } + } public void close() { _curator.close(); } - /** - * - * [zk: localhost:2181(CONNECTED) 56] get /brokers/ids/0 - * { "host":"localhost", "jmx_port":9999, "port":9092, "version":1 } - * - * @param contents - * @return - */ + /** + * [zk: localhost:2181(CONNECTED) 56] get /brokers/ids/0 + * { "host":"localhost", "jmx_port":9999, "port":9092, "version":1 } + * + * @param contents + * @return + */ private HostPort getBrokerHost(byte[] contents) { try { - Map value = (Map) JSONValue.parse(new String(contents, "UTF-8")); - String host = (String) value.get("host"); - Integer port = ((Long) value.get("port")).intValue(); + Map value = (Map) JSONValue.parse(new String(contents, "UTF-8")); + String host = (String) value.get("host"); + Integer port = ((Long) value.get("port")).intValue(); return new HostPort(host, port); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } - } + } } diff --git a/src/jvm/storm/kafka/DynamicPartitionConnections.java b/src/jvm/storm/kafka/DynamicPartitionConnections.java index 409ec3717a..7a799a0f1d 100644 --- a/src/jvm/storm/kafka/DynamicPartitionConnections.java +++ b/src/jvm/storm/kafka/DynamicPartitionConnections.java @@ -13,33 +13,33 @@ public class DynamicPartitionConnections { - public static final Logger LOG = LoggerFactory.getLogger(DynamicPartitionConnections.class); + public static final Logger LOG = LoggerFactory.getLogger(DynamicPartitionConnections.class); static class ConnectionInfo { SimpleConsumer consumer; Set partitions = new HashSet(); - + public ConnectionInfo(SimpleConsumer consumer) { this.consumer = consumer; } } - + Map _connections = new HashMap(); KafkaConfig _config; - IBrokerReader _reader; - + IBrokerReader _reader; + public DynamicPartitionConnections(KafkaConfig config, IBrokerReader brokerReader) { _config = config; - _reader = brokerReader; + _reader = brokerReader; } - + public SimpleConsumer register(Partition partition) { - HostPort hostPort = _reader.getCurrentBrokers().getHostFor(partition.partition); - return register(hostPort, partition.partition); + HostPort hostPort = _reader.getCurrentBrokers().getHostFor(partition.partition); + return register(hostPort, partition.partition); } - + public SimpleConsumer register(HostPort host, int partition) { - if(!_connections.containsKey(host)) { + if (!_connections.containsKey(host)) { _connections.put(host, new ConnectionInfo(new SimpleConsumer(host.host, host.port, _config.socketTimeoutMs, _config.bufferSizeBytes, _config.clientId))); } ConnectionInfo info = _connections.get(host); @@ -49,14 +49,16 @@ public SimpleConsumer register(HostPort host, int partition) { public SimpleConsumer getConnection(Partition partition) { ConnectionInfo info = _connections.get(partition.host); - if(info != null) return info.consumer; + if (info != null) { + return info.consumer; + } return null; } - + public void unregister(HostPort port, int partition) { ConnectionInfo info = _connections.get(port); info.partitions.remove(partition); - if(info.partitions.isEmpty()) { + if (info.partitions.isEmpty()) { info.consumer.close(); _connections.remove(port); } @@ -65,9 +67,9 @@ public void unregister(HostPort port, int partition) { public void unregister(Partition partition) { unregister(partition.host, partition.partition); } - + public void clear() { - for(ConnectionInfo info: _connections.values()) { + for (ConnectionInfo info : _connections.values()) { info.consumer.close(); } _connections.clear(); diff --git a/src/jvm/storm/kafka/HostPort.java b/src/jvm/storm/kafka/HostPort.java index afb5da527a..53698584ec 100644 --- a/src/jvm/storm/kafka/HostPort.java +++ b/src/jvm/storm/kafka/HostPort.java @@ -1,18 +1,16 @@ package storm.kafka; import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; public class HostPort implements Serializable, Comparable { public String host; public int port; - + public HostPort(String host, int port) { this.host = host; this.port = port; } - + public HostPort(String host) { this(host, 9092); } @@ -33,26 +31,26 @@ public String toString() { return host + ":" + port; } - public static HostPort fromString(String host) { - HostPort hp; - String[] spec = host.split(":"); - if (spec.length == 1) { - hp = new HostPort(spec[0]); - } else if (spec.length == 2) { - hp = new HostPort(spec[0], Integer.parseInt(spec[1])); - } else { - throw new IllegalArgumentException("Invalid host specification: " + host); - } - return hp; - } - - - @Override - public int compareTo(HostPort o) { - if ( this.host.equals(o.host)) { - return this.port - o.port; - } else { - return this.host.compareTo(o.host); - } - } + public static HostPort fromString(String host) { + HostPort hp; + String[] spec = host.split(":"); + if (spec.length == 1) { + hp = new HostPort(spec[0]); + } else if (spec.length == 2) { + hp = new HostPort(spec[0], Integer.parseInt(spec[1])); + } else { + throw new IllegalArgumentException("Invalid host specification: " + host); + } + return hp; + } + + + @Override + public int compareTo(HostPort o) { + if (this.host.equals(o.host)) { + return this.port - o.port; + } else { + return this.host.compareTo(o.host); + } + } } diff --git a/src/jvm/storm/kafka/KafkaConfig.java b/src/jvm/storm/kafka/KafkaConfig.java index 457eeb4620..e24197854a 100644 --- a/src/jvm/storm/kafka/KafkaConfig.java +++ b/src/jvm/storm/kafka/KafkaConfig.java @@ -9,11 +9,11 @@ public class KafkaConfig implements Serializable { public final BrokerHosts hosts; public final String topic; - public final String clientId; + public final String clientId; - public int fetchSizeBytes = 1024*1024; + public int fetchSizeBytes = 1024 * 1024; public int socketTimeoutMs = 10000; - public int bufferSizeBytes = 1024*1024; + public int bufferSizeBytes = 1024 * 1024; public MultiScheme scheme = new RawMultiScheme(); public boolean forceFromStart = false; public long startOffsetTime = kafka.api.OffsetRequest.EarliestTime(); @@ -22,11 +22,11 @@ public KafkaConfig(BrokerHosts hosts, String topic) { this(hosts, topic, kafka.api.OffsetRequest.DefaultClientId()); } - public KafkaConfig(BrokerHosts hosts, String topic, String clientId) { - this.hosts = hosts; - this.topic = topic; - this.clientId = clientId; - } + public KafkaConfig(BrokerHosts hosts, String topic, String clientId) { + this.hosts = hosts; + this.topic = topic; + this.clientId = clientId; + } public void forceStartOffsetTime(long millis) { startOffsetTime = millis; diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index 781e6ceb09..cf407ad577 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -54,21 +54,25 @@ public KafkaSpout(SpoutConfig spoutConf) { public void open(Map conf, final TopologyContext context, final SpoutOutputCollector collector) { _collector = collector; - Map stateConf = new HashMap(conf); + Map stateConf = new HashMap(conf); List zkServers = _spoutConfig.zkServers; - if(zkServers==null) zkServers = (List) conf.get(Config.STORM_ZOOKEEPER_SERVERS); + if (zkServers == null) { + zkServers = (List) conf.get(Config.STORM_ZOOKEEPER_SERVERS); + } Integer zkPort = _spoutConfig.zkPort; - if(zkPort==null) zkPort = ((Number) conf.get(Config.STORM_ZOOKEEPER_PORT)).intValue(); + if (zkPort == null) { + zkPort = ((Number) conf.get(Config.STORM_ZOOKEEPER_PORT)).intValue(); + } stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS, zkServers); stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_PORT, zkPort); stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_ROOT, _spoutConfig.zkRoot); - _state = new ZkState(stateConf); + _state = new ZkState(stateConf); _connections = new DynamicPartitionConnections(_spoutConfig, KafkaUtils.makeBrokerReader(conf, _spoutConfig)); // using TransactionalState like this is a hack int totalTasks = context.getComponentTasks(context.getThisComponentId()).size(); - if(_spoutConfig.hosts instanceof StaticHosts) { + if (_spoutConfig.hosts instanceof StaticHosts) { _coordinator = new StaticCoordinator(_connections, conf, _spoutConfig, _state, context.getThisTaskIndex(), totalTasks, _uuid); } else { _coordinator = new ZkCoordinator(_connections, conf, _spoutConfig, _state, context.getThisTaskIndex(), totalTasks, _uuid); @@ -76,13 +80,16 @@ public void open(Map conf, final TopologyContext context, final SpoutOutputColle context.registerMetric("kafkaOffset", new IMetric() { KafkaUtils.KafkaOffsetMetric _kafkaOffsetMetric = new KafkaUtils.KafkaOffsetMetric(_spoutConfig.topic, _connections); + @Override public Object getValueAndReset() { List pms = _coordinator.getMyManagedPartitions(); Set latestPartitions = new HashSet(); - for(PartitionManager pm : pms) { latestPartitions.add(pm.getPartition()); } + for (PartitionManager pm : pms) { + latestPartitions.add(pm.getPartition()); + } _kafkaOffsetMetric.refreshPartitions(latestPartitions); - for(PartitionManager pm : pms) { + for (PartitionManager pm : pms) { _kafkaOffsetMetric.setLatestEmittedOffset(pm.getPartition(), pm.lastCompletedOffset()); } return _kafkaOffsetMetric.getValueAndReset(); @@ -94,7 +101,7 @@ public Object getValueAndReset() { public Object getValueAndReset() { List pms = _coordinator.getMyManagedPartitions(); Map concatMetricsDataMaps = new HashMap(); - for(PartitionManager pm : pms) { + for (PartitionManager pm : pms) { concatMetricsDataMaps.putAll(pm.getMetricsDataMap()); } return concatMetricsDataMaps; @@ -104,27 +111,27 @@ public Object getValueAndReset() { @Override public void close() { - _state.close(); + _state.close(); } @Override public void nextTuple() { List managers = _coordinator.getMyManagedPartitions(); - for(int i=0; i _spoutConfig.stateUpdateIntervalMs) { + if ((now - _lastUpdateMs) > _spoutConfig.stateUpdateIntervalMs) { commit(); } } @@ -133,18 +140,18 @@ public void nextTuple() { public void ack(Object msgId) { KafkaMessageId id = (KafkaMessageId) msgId; PartitionManager m = _coordinator.getManager(id.partition); - if(m!=null) { + if (m != null) { m.ack(id.offset); - } + } } @Override public void fail(Object msgId) { KafkaMessageId id = (KafkaMessageId) msgId; PartitionManager m = _coordinator.getManager(id.partition); - if(m!=null) { + if (m != null) { m.fail(id.offset); - } + } } @Override @@ -159,7 +166,7 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { private void commit() { _lastUpdateMs = System.currentTimeMillis(); - for(PartitionManager manager: _coordinator.getMyManagedPartitions()) { + for (PartitionManager manager : _coordinator.getMyManagedPartitions()) { manager.commit(); } } diff --git a/src/jvm/storm/kafka/Partition.java b/src/jvm/storm/kafka/Partition.java index 6de0f6a1fd..87ab7b8128 100644 --- a/src/jvm/storm/kafka/Partition.java +++ b/src/jvm/storm/kafka/Partition.java @@ -6,7 +6,7 @@ public class Partition implements ISpoutPartition { - public final HostPort host; + public final HostPort host; public final int partition; public Partition(HostPort host, int partition) { @@ -14,30 +14,30 @@ public Partition(HostPort host, int partition) { this.partition = partition; } - @Override - public int hashCode() { - return Objects.hashCode(host, partition); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final Partition other = (Partition) obj; - return Objects.equal(this.host, other.host) && Objects.equal(this.partition, other.partition); - } - - @Override - public String toString() { - return "Partition{" + - "host=" + host + - ", partition=" + partition + - '}'; - } + @Override + public int hashCode() { + return Objects.hashCode(host, partition); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final Partition other = (Partition) obj; + return Objects.equal(this.host, other.host) && Objects.equal(this.partition, other.partition); + } + + @Override + public String toString() { + return "Partition{" + + "host=" + host + + ", partition=" + partition + + '}'; + } @Override public String getId() { diff --git a/src/jvm/storm/kafka/PartitionCoordinator.java b/src/jvm/storm/kafka/PartitionCoordinator.java index 2ee2009e55..d28248d1ec 100644 --- a/src/jvm/storm/kafka/PartitionCoordinator.java +++ b/src/jvm/storm/kafka/PartitionCoordinator.java @@ -4,5 +4,6 @@ public interface PartitionCoordinator { List getMyManagedPartitions(); + PartitionManager getManager(Partition partition); } diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 6596f9674b..623bc10fe1 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -58,31 +58,30 @@ public PartitionManager(DynamicPartitionConnections connections, String topology _spoutConfig = spoutConfig; _topologyInstanceId = topologyInstanceId; _consumer = connections.register(id.host, id.partition); - _state = state; + _state = state; _stormConf = stormConf; String jsonTopologyId = null; Long jsonOffset = null; try { Map json = _state.readJSON(committedPath()); - if(json != null) { - jsonTopologyId = (String)((Map)json.get("topology")).get("id"); - jsonOffset = (Long)json.get("offset"); + if (json != null) { + jsonTopologyId = (String) ((Map) json.get("topology")).get("id"); + jsonOffset = (Long) json.get("offset"); } - } - catch(Throwable e) { + } catch (Throwable e) { LOG.warn("Error reading and/or parsing at ZkNode: " + committedPath(), e); } - if(!topologyInstanceId.equals(jsonTopologyId) && spoutConfig.forceFromStart) { + if (!topologyInstanceId.equals(jsonTopologyId) && spoutConfig.forceFromStart) { _committedTo = KafkaUtils.getOffset(_consumer, spoutConfig.topic, id.partition, spoutConfig.startOffsetTime); - LOG.info("Using startOffsetTime to choose last commit offset."); - } else if(jsonTopologyId == null || jsonOffset == null) { // failed to parse JSON? - _committedTo = KafkaUtils.getOffset(_consumer, spoutConfig.topic, id.partition, kafka.api.OffsetRequest.LatestTime()); - LOG.info("Setting last commit offset to HEAD."); + LOG.info("Using startOffsetTime to choose last commit offset."); + } else if (jsonTopologyId == null || jsonOffset == null) { // failed to parse JSON? + _committedTo = KafkaUtils.getOffset(_consumer, spoutConfig.topic, id.partition, kafka.api.OffsetRequest.LatestTime()); + LOG.info("Setting last commit offset to HEAD."); } else { _committedTo = jsonOffset; - LOG.info("Read last commit offset from zookeeper: " + _committedTo); + LOG.info("Read last commit offset from zookeeper: " + _committedTo); } LOG.info("Starting Kafka " + _consumer.host() + ":" + id.partition + " from offset " + _committedTo); @@ -105,22 +104,25 @@ public Map getMetricsDataMap() { //returns false if it's reached the end of current batch public EmitState next(SpoutOutputCollector collector) { - if(_waitingToEmit.isEmpty()) fill(); - while(true) { + if (_waitingToEmit.isEmpty()) { + fill(); + } + while (true) { MessageAndRealOffset toEmit = _waitingToEmit.pollFirst(); - if(toEmit==null) { + if (toEmit == null) { return EmitState.NO_EMITTED; } Iterable> tups = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.msg.payload())); - if(tups!=null) { - for(List tup: tups) + if (tups != null) { + for (List tup : tups) { collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset)); + } break; } else { ack(toEmit.offset); } } - if(!_waitingToEmit.isEmpty()) { + if (!_waitingToEmit.isEmpty()) { return EmitState.EMITTED_MORE_LEFT; } else { return EmitState.EMITTED_END; @@ -132,11 +134,11 @@ private void fill() { long start = System.nanoTime(); ByteBufferMessageSet msgs = _consumer.fetch( new FetchRequestBuilder().addFetch( - _spoutConfig.topic, - _partition.partition, - _emittedToOffset, - _spoutConfig.fetchSizeBytes).build()).messageSet(_spoutConfig.topic, - _partition.partition); + _spoutConfig.topic, + _partition.partition, + _emittedToOffset, + _spoutConfig.fetchSizeBytes).build()).messageSet(_spoutConfig.topic, + _partition.partition); long end = System.nanoTime(); long millis = (end - start) / 1000000; _fetchAPILatencyMax.update(millis); @@ -145,26 +147,26 @@ private void fill() { int numMessages = countMessages(msgs); _fetchAPIMessageCount.incrBy(numMessages); - if(numMessages>0) { - LOG.info("Fetched " + numMessages + " messages from Kafka: " + _consumer.host() + ":" + _partition.partition); + if (numMessages > 0) { + LOG.info("Fetched " + numMessages + " messages from Kafka: " + _consumer.host() + ":" + _partition.partition); } - for(MessageAndOffset msg: msgs) { + for (MessageAndOffset msg : msgs) { _pending.add(_emittedToOffset); _waitingToEmit.add(new MessageAndRealOffset(msg.message(), _emittedToOffset)); _emittedToOffset = msg.nextOffset(); } - if(numMessages>0) { - LOG.info("Added " + numMessages + " messages from Kafka: " + _consumer.host() + ":" + _partition.partition + " to internal buffers"); + if (numMessages > 0) { + LOG.info("Added " + numMessages + " messages from Kafka: " + _consumer.host() + ":" + _partition.partition + " to internal buffers"); } } - private int countMessages(ByteBufferMessageSet messageSet) { - int counter = 0; - for (MessageAndOffset messageAndOffset : messageSet) { - counter = counter + 1; - } - return counter; - } + private int countMessages(ByteBufferMessageSet messageSet) { + int counter = 0; + for (MessageAndOffset messageAndOffset : messageSet) { + counter = counter + 1; + } + return counter; + } public void ack(Long offset) { _pending.remove(offset); @@ -173,7 +175,7 @@ public void ack(Long offset) { public void fail(Long offset) { //TODO: should it use in-memory ack set to skip anything that's been acked but not committed??? // things might get crazy with lots of timeouts - if(_emittedToOffset > offset) { + if (_emittedToOffset > offset) { _emittedToOffset = offset; _pending.tailSet(offset).clear(); } @@ -182,23 +184,23 @@ public void fail(Long offset) { public void commit() { LOG.info("Committing offset for " + _partition); long committedTo; - if(_pending.isEmpty()) { + if (_pending.isEmpty()) { committedTo = _emittedToOffset; } else { committedTo = _pending.first(); } - if(committedTo!=_committedTo) { + if (committedTo != _committedTo) { LOG.info("Writing committed offset to ZK: " + committedTo); - Map data = (Map)ImmutableMap.builder() - .put("topology", ImmutableMap.of("id", _topologyInstanceId, - "name", _stormConf.get(Config.TOPOLOGY_NAME))) - .put("offset", committedTo) - .put("partition", _partition.partition) - .put("broker", ImmutableMap.of("host", _partition.host.host, - "port", _partition.host.port)) - .put("topic", _spoutConfig.topic).build(); - _state.writeJSON(committedPath(), data); + Map data = (Map) ImmutableMap.builder() + .put("topology", ImmutableMap.of("id", _topologyInstanceId, + "name", _stormConf.get(Config.TOPOLOGY_NAME))) + .put("offset", committedTo) + .put("partition", _partition.partition) + .put("broker", ImmutableMap.of("host", _partition.host.host, + "port", _partition.host.port)) + .put("topic", _spoutConfig.topic).build(); + _state.writeJSON(committedPath(), data); LOG.info("Wrote committed offset to ZK: " + committedTo); _committedTo = committedTo; @@ -212,7 +214,7 @@ private String committedPath() { public long queryPartitionOffsetLatestTime() { return KafkaUtils.getOffset(_consumer, _spoutConfig.topic, _partition.partition, - OffsetRequest.LatestTime()); + OffsetRequest.LatestTime()); } public long lastCommittedOffset() { @@ -220,7 +222,7 @@ public long lastCommittedOffset() { } public long lastCompletedOffset() { - if(_pending.isEmpty()) { + if (_pending.isEmpty()) { return _emittedToOffset; } else { return _pending.first(); diff --git a/src/jvm/storm/kafka/StaticCoordinator.java b/src/jvm/storm/kafka/StaticCoordinator.java index 6f97c8b727..741552242b 100644 --- a/src/jvm/storm/kafka/StaticCoordinator.java +++ b/src/jvm/storm/kafka/StaticCoordinator.java @@ -13,22 +13,22 @@ public class StaticCoordinator implements PartitionCoordinator { public StaticCoordinator(DynamicPartitionConnections connections, Map stormConf, SpoutConfig config, ZkState state, int taskIndex, int totalTasks, String topologyInstanceId) { StaticHosts hosts = (StaticHosts) config.hosts; List partitions = hosts.getPartitionInformation().getOrderedPartitions(); - for(int i=taskIndex; i getMyManagedPartitions() { return _allManagers; } - + public PartitionManager getManager(Partition partition) { return _managers.get(partition); } - + } diff --git a/src/jvm/storm/kafka/StaticHosts.java b/src/jvm/storm/kafka/StaticHosts.java index 6ed828d5d5..9ed719388f 100644 --- a/src/jvm/storm/kafka/StaticHosts.java +++ b/src/jvm/storm/kafka/StaticHosts.java @@ -2,9 +2,6 @@ import storm.kafka.trident.GlobalPartitionInformation; -import java.util.ArrayList; -import java.util.List; - /** * Date: 11/05/2013 * Time: 14:43 @@ -12,13 +9,13 @@ public class StaticHosts implements BrokerHosts { - private GlobalPartitionInformation partitionInformation; + private GlobalPartitionInformation partitionInformation; - public StaticHosts(GlobalPartitionInformation partitionInformation) { - this.partitionInformation = partitionInformation; - } + public StaticHosts(GlobalPartitionInformation partitionInformation) { + this.partitionInformation = partitionInformation; + } - public GlobalPartitionInformation getPartitionInformation() { - return partitionInformation; - } + public GlobalPartitionInformation getPartitionInformation() { + return partitionInformation; + } } diff --git a/src/jvm/storm/kafka/StaticPartitionConnections.java b/src/jvm/storm/kafka/StaticPartitionConnections.java index 2d40c8b67b..4294362f9f 100644 --- a/src/jvm/storm/kafka/StaticPartitionConnections.java +++ b/src/jvm/storm/kafka/StaticPartitionConnections.java @@ -9,17 +9,17 @@ public class StaticPartitionConnections { Map _kafka = new HashMap(); KafkaConfig _config; StaticHosts hosts; - + public StaticPartitionConnections(KafkaConfig conf) { _config = conf; - if(!(conf.hosts instanceof StaticHosts)) { + if (!(conf.hosts instanceof StaticHosts)) { throw new RuntimeException("Must configure with static hosts"); } this.hosts = (StaticHosts) conf.hosts; } public SimpleConsumer getConsumer(int partition) { - if(!_kafka.containsKey(partition)) { + if (!_kafka.containsKey(partition)) { HostPort hp = hosts.getPartitionInformation().getHostFor(partition); _kafka.put(partition, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes, _config.clientId)); @@ -28,7 +28,7 @@ public SimpleConsumer getConsumer(int partition) { } public void close() { - for(SimpleConsumer consumer: _kafka.values()) { + for (SimpleConsumer consumer : _kafka.values()) { consumer.close(); } } diff --git a/src/jvm/storm/kafka/ZkCoordinator.java b/src/jvm/storm/kafka/ZkCoordinator.java index d457bddbaf..98e51a3170 100644 --- a/src/jvm/storm/kafka/ZkCoordinator.java +++ b/src/jvm/storm/kafka/ZkCoordinator.java @@ -9,7 +9,7 @@ public class ZkCoordinator implements PartitionCoordinator { public static final Logger LOG = LoggerFactory.getLogger(ZkCoordinator.class); - + SpoutConfig _spoutConfig; int _taskIndex; int _totalTasks; @@ -23,7 +23,7 @@ public class ZkCoordinator implements PartitionCoordinator { ZkState _state; Map _stormConf; IMetricsContext _metricsContext; - + public ZkCoordinator(DynamicPartitionConnections connections, Map stormConf, SpoutConfig spoutConfig, ZkState state, int taskIndex, int totalTasks, String topologyInstanceId) { _spoutConfig = spoutConfig; _connections = connections; @@ -31,55 +31,55 @@ public ZkCoordinator(DynamicPartitionConnections connections, Map stormConf, Spo _totalTasks = totalTasks; _topologyInstanceId = topologyInstanceId; _stormConf = stormConf; - _state = state; + _state = state; ZkHosts brokerConf = (ZkHosts) spoutConfig.hosts; _refreshFreqMs = brokerConf.refreshFreqSecs * 1000; _reader = new DynamicBrokersReader(stormConf, brokerConf.brokerZkStr, brokerConf.brokerZkPath, spoutConfig.topic); - + } - + @Override public List getMyManagedPartitions() { - if(_lastRefreshTime==null || (System.currentTimeMillis() - _lastRefreshTime) > _refreshFreqMs) { + if (_lastRefreshTime == null || (System.currentTimeMillis() - _lastRefreshTime) > _refreshFreqMs) { refresh(); _lastRefreshTime = System.currentTimeMillis(); } return _cachedList; } - + void refresh() { try { LOG.info("Refreshing partition manager connections"); - GlobalPartitionInformation brokerInfo = _reader.getBrokerInfo(); + GlobalPartitionInformation brokerInfo = _reader.getBrokerInfo(); Set mine = new HashSet(); - for (Partition partitionId: brokerInfo){ - if(myOwnership(partitionId)) { - mine.add(partitionId); - } - } + for (Partition partitionId : brokerInfo) { + if (myOwnership(partitionId)) { + mine.add(partitionId); + } + } Set curr = _managers.keySet(); Set newPartitions = new HashSet(mine); newPartitions.removeAll(curr); - + Set deletedPartitions = new HashSet(curr); deletedPartitions.removeAll(mine); - + LOG.info("Deleted partition managers: " + deletedPartitions.toString()); - - for(Partition id: deletedPartitions) { + + for (Partition id : deletedPartitions) { PartitionManager man = _managers.remove(id); man.close(); } LOG.info("New partition managers: " + newPartitions.toString()); - - for(Partition id: newPartitions) { + + for (Partition id : newPartitions) { PartitionManager man = new PartitionManager(_connections, _topologyInstanceId, _state, _stormConf, _spoutConfig, id); _managers.put(id, man); } - - } catch(Exception e) { + + } catch (Exception e) { throw new RuntimeException(e); } _cachedList = new ArrayList(_managers.values()); @@ -90,9 +90,9 @@ void refresh() { public PartitionManager getManager(Partition partition) { return _managers.get(partition); } - + private boolean myOwnership(Partition id) { - int val = Math.abs(id.host.hashCode() + 23 * id.partition); + int val = Math.abs(id.host.hashCode() + 23 * id.partition); return val % _totalTasks == _taskIndex; - } + } } diff --git a/src/jvm/storm/kafka/ZkHosts.java b/src/jvm/storm/kafka/ZkHosts.java index dfe3b4cdc5..f2e0fc24b3 100644 --- a/src/jvm/storm/kafka/ZkHosts.java +++ b/src/jvm/storm/kafka/ZkHosts.java @@ -5,18 +5,18 @@ * Time: 14:38 */ public class ZkHosts implements BrokerHosts { - private static final String DEFAULT_ZK_PATH = "/brokers"; + private static final String DEFAULT_ZK_PATH = "/brokers"; - public String brokerZkStr = null; - public String brokerZkPath = null; // e.g., /kafka/brokers - public int refreshFreqSecs = 60; + public String brokerZkStr = null; + public String brokerZkPath = null; // e.g., /kafka/brokers + public int refreshFreqSecs = 60; - public ZkHosts(String brokerZkStr, String brokerZkPath) { - this.brokerZkStr = brokerZkStr; - this.brokerZkPath = brokerZkPath; - } + public ZkHosts(String brokerZkStr, String brokerZkPath) { + this.brokerZkStr = brokerZkStr; + this.brokerZkPath = brokerZkPath; + } - public ZkHosts(String brokerZkStr) { - this(brokerZkStr, DEFAULT_ZK_PATH); - } + public ZkHosts(String brokerZkStr) { + this(brokerZkStr, DEFAULT_ZK_PATH); + } } \ No newline at end of file diff --git a/src/jvm/storm/kafka/ZkState.java b/src/jvm/storm/kafka/ZkState.java index 17ebb25286..d5416af406 100644 --- a/src/jvm/storm/kafka/ZkState.java +++ b/src/jvm/storm/kafka/ZkState.java @@ -20,78 +20,80 @@ public class ZkState { CuratorFramework _curator; private CuratorFramework newCurator(Map stateConf) throws Exception { - Integer port = (Integer)stateConf.get(Config.TRANSACTIONAL_ZOOKEEPER_PORT); - String serverPorts = ""; - for(String server: (List)stateConf.get(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS)) { + Integer port = (Integer) stateConf.get(Config.TRANSACTIONAL_ZOOKEEPER_PORT); + String serverPorts = ""; + for (String server : (List) stateConf.get(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS)) { serverPorts = serverPorts + server + ":" + port + ","; } - return CuratorFrameworkFactory.newClient(serverPorts, - Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), - 15000, - new RetryNTimes(Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), - Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); + return CuratorFrameworkFactory.newClient(serverPorts, + Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), + 15000, + new RetryNTimes(Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), + Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); } public CuratorFramework getCurator() { - assert _curator != null; + assert _curator != null; return _curator; } public ZkState(Map stateConf) { - stateConf = new HashMap(stateConf); + stateConf = new HashMap(stateConf); - try { - _curator = newCurator(stateConf); - _curator.start(); - } catch(Exception e) { - throw new RuntimeException(e); - } + try { + _curator = newCurator(stateConf); + _curator.start(); + } catch (Exception e) { + throw new RuntimeException(e); + } } - public void writeJSON(String path, Map data) { - LOG.info("Writing " + path + " the data " + data.toString()); + public void writeJSON(String path, Map data) { + LOG.info("Writing " + path + " the data " + data.toString()); writeBytes(path, JSONValue.toJSONString(data).getBytes(Charset.forName("UTF-8"))); } public void writeBytes(String path, byte[] bytes) { try { - if(_curator.checkExists().forPath(path)==null) { + if (_curator.checkExists().forPath(path) == null) { _curator.create() .creatingParentsIfNeeded() .withMode(CreateMode.PERSISTENT) .forPath(path, bytes); } else { - _curator.setData().forPath(path, bytes); - } - } catch(Exception e) { + _curator.setData().forPath(path, bytes); + } + } catch (Exception e) { throw new RuntimeException(e); } } - public Map readJSON(String path) { - try { - byte[] b = readBytes(path); - if(b==null) return null; - return (Map)JSONValue.parse(new String(b, "UTF-8")); - } catch(Exception e) { - throw new RuntimeException(e); - } + public Map readJSON(String path) { + try { + byte[] b = readBytes(path); + if (b == null) { + return null; + } + return (Map) JSONValue.parse(new String(b, "UTF-8")); + } catch (Exception e) { + throw new RuntimeException(e); + } } public byte[] readBytes(String path) { try { - if(_curator.checkExists().forPath(path)!=null) { - return _curator.getData().forPath(path); + if (_curator.checkExists().forPath(path) != null) { + return _curator.getData().forPath(path); } else { return null; } - } catch(Exception e) { + } catch (Exception e) { throw new RuntimeException(e); } } public void close() { - _curator.close(); - _curator = null; + _curator.close(); + _curator = null; } } diff --git a/src/jvm/storm/kafka/trident/Coordinator.java b/src/jvm/storm/kafka/trident/Coordinator.java index b7ddd3f08b..d97feedaa7 100644 --- a/src/jvm/storm/kafka/trident/Coordinator.java +++ b/src/jvm/storm/kafka/trident/Coordinator.java @@ -11,26 +11,26 @@ */ class Coordinator implements IPartitionedTridentSpout.Coordinator, IOpaquePartitionedTridentSpout.Coordinator { - private IBrokerReader reader; - private TridentKafkaConfig config; - - public Coordinator(Map conf, TridentKafkaConfig tridentKafkaConfig) { - config = tridentKafkaConfig; - reader = KafkaUtils.makeBrokerReader(conf, config); - } - - @Override - public void close() { - config.coordinator.close(); - } - - @Override - public boolean isReady(long txid) { - return config.coordinator.isReady(txid); - } - - @Override - public GlobalPartitionInformation getPartitionsForBatch() { - return reader.getCurrentBrokers(); - } + private IBrokerReader reader; + private TridentKafkaConfig config; + + public Coordinator(Map conf, TridentKafkaConfig tridentKafkaConfig) { + config = tridentKafkaConfig; + reader = KafkaUtils.makeBrokerReader(conf, config); + } + + @Override + public void close() { + config.coordinator.close(); + } + + @Override + public boolean isReady(long txid) { + return config.coordinator.isReady(txid); + } + + @Override + public GlobalPartitionInformation getPartitionsForBatch() { + return reader.getCurrentBrokers(); + } } diff --git a/src/jvm/storm/kafka/trident/DefaultCoordinator.java b/src/jvm/storm/kafka/trident/DefaultCoordinator.java index 3a477062cf..89cd5034ac 100644 --- a/src/jvm/storm/kafka/trident/DefaultCoordinator.java +++ b/src/jvm/storm/kafka/trident/DefaultCoordinator.java @@ -10,5 +10,5 @@ public boolean isReady(long txid) { @Override public void close() { } - + } diff --git a/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java b/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java index 139a0d783d..6b0fdec90a 100644 --- a/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java +++ b/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java @@ -1,7 +1,7 @@ package storm.kafka.trident; -import storm.kafka.Partition; import storm.kafka.HostPort; +import storm.kafka.Partition; import java.io.Serializable; import java.util.*; @@ -12,55 +12,55 @@ */ public class GlobalPartitionInformation implements Iterable, Serializable { - private Map partitionMap; + private Map partitionMap; - public GlobalPartitionInformation() { - partitionMap = new TreeMap(); - } + public GlobalPartitionInformation() { + partitionMap = new TreeMap(); + } - public void addPartition(int partitionId, HostPort broker) { - partitionMap.put(partitionId, broker); - } + public void addPartition(int partitionId, HostPort broker) { + partitionMap.put(partitionId, broker); + } - @Override - public String toString() { - return "GlobalPartitionInformation{" + - "partitionMap=" + partitionMap + - '}'; - } + @Override + public String toString() { + return "GlobalPartitionInformation{" + + "partitionMap=" + partitionMap + + '}'; + } - public HostPort getHostFor(Integer partitionId) { - return partitionMap.get(partitionId); - } + public HostPort getHostFor(Integer partitionId) { + return partitionMap.get(partitionId); + } - public List getOrderedPartitions(){ - List partitions = new LinkedList(); - for (Map.Entry partition : partitionMap.entrySet()) { - partitions.add(new Partition(partition.getValue(), partition.getKey())); - } - return partitions; - } + public List getOrderedPartitions() { + List partitions = new LinkedList(); + for (Map.Entry partition : partitionMap.entrySet()) { + partitions.add(new Partition(partition.getValue(), partition.getKey())); + } + return partitions; + } - @Override - public Iterator iterator() { - final Iterator> iterator = partitionMap.entrySet().iterator(); + @Override + public Iterator iterator() { + final Iterator> iterator = partitionMap.entrySet().iterator(); - return new Iterator() { - @Override - public boolean hasNext() { - return iterator.hasNext(); - } + return new Iterator() { + @Override + public boolean hasNext() { + return iterator.hasNext(); + } - @Override - public Partition next() { - Map.Entry next = iterator.next(); - return new Partition(next.getValue(), next.getKey()); - } + @Override + public Partition next() { + Map.Entry next = iterator.next(); + return new Partition(next.getValue(), next.getKey()); + } - @Override - public void remove() { - iterator.remove(); - } - }; - } + @Override + public void remove() { + iterator.remove(); + } + }; + } } diff --git a/src/jvm/storm/kafka/trident/IBatchCoordinator.java b/src/jvm/storm/kafka/trident/IBatchCoordinator.java index 9199a8da1b..1b8a8ce337 100644 --- a/src/jvm/storm/kafka/trident/IBatchCoordinator.java +++ b/src/jvm/storm/kafka/trident/IBatchCoordinator.java @@ -4,5 +4,6 @@ public interface IBatchCoordinator extends Serializable { boolean isReady(long txid); + void close(); } diff --git a/src/jvm/storm/kafka/trident/IBrokerReader.java b/src/jvm/storm/kafka/trident/IBrokerReader.java index 4e2421b1c9..73c9738ea8 100644 --- a/src/jvm/storm/kafka/trident/IBrokerReader.java +++ b/src/jvm/storm/kafka/trident/IBrokerReader.java @@ -3,5 +3,6 @@ public interface IBrokerReader { GlobalPartitionInformation getCurrentBrokers(); + void close(); } diff --git a/src/jvm/storm/kafka/trident/KafkaUtils.java b/src/jvm/storm/kafka/trident/KafkaUtils.java index efe4fefdb7..18dd851fb2 100644 --- a/src/jvm/storm/kafka/trident/KafkaUtils.java +++ b/src/jvm/storm/kafka/trident/KafkaUtils.java @@ -16,33 +16,33 @@ public class KafkaUtils { public static final Logger LOG = LoggerFactory.getLogger(KafkaUtils.class); - private static final int NO_OFFSET = -5; + private static final int NO_OFFSET = -5; - public static IBrokerReader makeBrokerReader(Map stormConf, KafkaConfig conf) { - if(conf.hosts instanceof StaticHosts) { - return new StaticBrokerReader(((StaticHosts) conf.hosts).getPartitionInformation()); - } else { - return new ZkBrokerReader(stormConf, conf.topic, (ZkHosts) conf.hosts); - } - } + public static IBrokerReader makeBrokerReader(Map stormConf, KafkaConfig conf) { + if (conf.hosts instanceof StaticHosts) { + return new StaticBrokerReader(((StaticHosts) conf.hosts).getPartitionInformation()); + } else { + return new ZkBrokerReader(stormConf, conf.topic, (ZkHosts) conf.hosts); + } + } - public static long getOffset(SimpleConsumer consumer, String topic, int partition, long startOffsetTime) { - TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); - Map requestInfo = new HashMap(); - requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1)); - OffsetRequest request = new OffsetRequest( - requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); + public static long getOffset(SimpleConsumer consumer, String topic, int partition, long startOffsetTime) { + TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); + Map requestInfo = new HashMap(); + requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1)); + OffsetRequest request = new OffsetRequest( + requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); - long[] offsets = consumer.getOffsetsBefore(request).offsets(topic, partition); - if ( offsets.length > 0) { - return offsets[0]; - } else { - return NO_OFFSET; - } - } + long[] offsets = consumer.getOffsetsBefore(request).offsets(topic, partition); + if (offsets.length > 0) { + return offsets[0]; + } else { + return NO_OFFSET; + } + } - public static class KafkaOffsetMetric implements IMetric { + public static class KafkaOffsetMetric implements IMetric { Map _partitionToOffset = new HashMap(); Set _partitions; String _topic; @@ -64,16 +64,16 @@ public Object getValueAndReset() { long totalLatestTimeOffset = 0; long totalLatestEmittedOffset = 0; HashMap ret = new HashMap(); - if(_partitions != null && _partitions.size() == _partitionToOffset.size()) { - for(Map.Entry e : _partitionToOffset.entrySet()) { + if (_partitions != null && _partitions.size() == _partitionToOffset.size()) { + for (Map.Entry e : _partitionToOffset.entrySet()) { Partition partition = e.getKey(); SimpleConsumer consumer = _connections.getConnection(partition); - if(consumer == null) { + if (consumer == null) { LOG.warn("partitionToOffset contains partition not found in _connections. Stale partition data?"); return null; } long latestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.LatestTime()); - if(latestTimeOffset == 0) { + if (latestTimeOffset == 0) { LOG.warn("No data found in Kafka Partition " + partition.getId()); return null; } @@ -93,18 +93,22 @@ public Object getValueAndReset() { } else { LOG.info("Metrics Tick: Not enough data to calculate spout lag."); } - } catch(Throwable t) { + } catch (Throwable t) { LOG.warn("Metrics Tick: Exception when computing kafkaOffset metric.", t); } return null; } - public void refreshPartitions(Set partitions) { - _partitions = partitions; - Iterator it = _partitionToOffset.keySet().iterator(); - while(it.hasNext()) { - if(!partitions.contains(it.next())) it.remove(); - } - } - }; + public void refreshPartitions(Set partitions) { + _partitions = partitions; + Iterator it = _partitionToOffset.keySet().iterator(); + while (it.hasNext()) { + if (!partitions.contains(it.next())) { + it.remove(); + } + } + } + } + + ; } diff --git a/src/jvm/storm/kafka/trident/MaxMetric.java b/src/jvm/storm/kafka/trident/MaxMetric.java index 087245f7e7..a8f88ba410 100644 --- a/src/jvm/storm/kafka/trident/MaxMetric.java +++ b/src/jvm/storm/kafka/trident/MaxMetric.java @@ -11,8 +11,12 @@ public Long identity() { @Override public Long combine(Long l1, Long l2) { - if(l1 == null) return l2; - if(l2 == null) return l1; + if (l1 == null) { + return l2; + } + if (l2 == null) { + return l1; + } return Math.max(l1, l2); } diff --git a/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java b/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java index 0f6e6c518b..35b7033ffb 100644 --- a/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java +++ b/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java @@ -11,19 +11,19 @@ public class OpaqueTridentKafkaSpout implements IOpaquePartitionedTridentSpout { - + TridentKafkaConfig _config; String _topologyInstanceId = UUID.randomUUID().toString(); - + public OpaqueTridentKafkaSpout(TridentKafkaConfig config) { _config = config; } - + @Override public IOpaquePartitionedTridentSpout.Emitter getEmitter(Map conf, TopologyContext context) { - return new TridentKafkaEmitter(conf, context, _config, _topologyInstanceId).asOpaqueEmitter(); + return new TridentKafkaEmitter(conf, context, _config, _topologyInstanceId).asOpaqueEmitter(); } - + @Override public IOpaquePartitionedTridentSpout.Coordinator getCoordinator(Map conf, TopologyContext tc) { return new storm.kafka.trident.Coordinator(conf, _config); @@ -32,8 +32,8 @@ public IOpaquePartitionedTridentSpout.Coordinator getCoordinator(Map conf, Topol @Override public Fields getOutputFields() { return _config.scheme.getOutputFields(); - } - + } + @Override public Map getComponentConfiguration() { return null; diff --git a/src/jvm/storm/kafka/trident/StaticBrokerReader.java b/src/jvm/storm/kafka/trident/StaticBrokerReader.java index dabbd5e7a0..98a8f53204 100644 --- a/src/jvm/storm/kafka/trident/StaticBrokerReader.java +++ b/src/jvm/storm/kafka/trident/StaticBrokerReader.java @@ -2,12 +2,12 @@ public class StaticBrokerReader implements IBrokerReader { - private GlobalPartitionInformation brokers = new GlobalPartitionInformation(); - + private GlobalPartitionInformation brokers = new GlobalPartitionInformation(); + public StaticBrokerReader(GlobalPartitionInformation partitionInformation) { this.brokers = partitionInformation; } - + @Override public GlobalPartitionInformation getCurrentBrokers() { return brokers; diff --git a/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java b/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java index 173a98f575..b32d301026 100644 --- a/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java +++ b/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java @@ -10,7 +10,7 @@ public class TransactionalTridentKafkaSpout implements IPartitionedTridentSpout { - + TridentKafkaConfig _config; String _topologyInstanceId = UUID.randomUUID().toString(); @@ -26,14 +26,14 @@ public IPartitionedTridentSpout.Coordinator getCoordinator(Map conf, TopologyCon @Override public IPartitionedTridentSpout.Emitter getEmitter(Map conf, TopologyContext context) { - return new TridentKafkaEmitter(conf, context, _config, _topologyInstanceId).asTransactionalEmitter(); + return new TridentKafkaEmitter(conf, context, _config, _topologyInstanceId).asTransactionalEmitter(); } @Override public Fields getOutputFields() { - return _config.scheme.getOutputFields(); + return _config.scheme.getOutputFields(); } - + @Override public Map getComponentConfiguration() { return null; diff --git a/src/jvm/storm/kafka/trident/TridentKafkaConfig.java b/src/jvm/storm/kafka/trident/TridentKafkaConfig.java index 71955009c1..073afa2592 100644 --- a/src/jvm/storm/kafka/trident/TridentKafkaConfig.java +++ b/src/jvm/storm/kafka/trident/TridentKafkaConfig.java @@ -13,8 +13,8 @@ public TridentKafkaConfig(BrokerHosts hosts, String topic) { super(hosts, topic); } - public TridentKafkaConfig(BrokerHosts hosts, String topic, String clientId) { - super(hosts, topic, clientId); - } + public TridentKafkaConfig(BrokerHosts hosts, String topic, String clientId) { + super(hosts, topic, clientId); + } } diff --git a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java index 282e67a1a1..ab4ec6393f 100644 --- a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java +++ b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java @@ -34,237 +34,241 @@ */ public class TridentKafkaEmitter { - public static final Logger LOG = LoggerFactory.getLogger(TridentKafkaEmitter.class); - - private DynamicPartitionConnections _connections; - private String _topologyName; - private KafkaUtils.KafkaOffsetMetric _kafkaOffsetMetric; - private ReducedMetric _kafkaMeanFetchLatencyMetric; - private CombinedMetric _kafkaMaxFetchLatencyMetric; - private TridentKafkaConfig _config; - private String _topologyInstanceId; - - public TridentKafkaEmitter(Map conf, TopologyContext context, TridentKafkaConfig config, String topologyInstanceId) { - _config = config; - _topologyInstanceId = topologyInstanceId; - _connections = new DynamicPartitionConnections(_config, KafkaUtils.makeBrokerReader(conf, _config)); - _topologyName = (String) conf.get(Config.TOPOLOGY_NAME); - _kafkaOffsetMetric = new KafkaUtils.KafkaOffsetMetric(_config.topic, _connections); - context.registerMetric("kafkaOffset", _kafkaOffsetMetric, 60); - _kafkaMeanFetchLatencyMetric = context.registerMetric("kafkaFetchAvg", new MeanReducer(), 60); - _kafkaMaxFetchLatencyMetric = context.registerMetric("kafkaFetchMax", new MaxMetric(), 60); - } - - - private Map failFastEmitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map lastMeta) { - SimpleConsumer consumer = _connections.register(partition); - Map ret = doEmitNewPartitionBatch(consumer, partition, collector, lastMeta); - _kafkaOffsetMetric.setLatestEmittedOffset(partition, (Long) ret.get("offset")); - return ret; - } - - private Map emitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map lastMeta) { - try { - return failFastEmitNewPartitionBatch(attempt, collector, partition, lastMeta); - } catch (FailedFetchException e) { - LOG.warn("Failed to fetch from partition " + partition); - if (lastMeta == null) { - return null; - } else { - Map ret = new HashMap(); - ret.put("offset", lastMeta.get("nextOffset")); - ret.put("nextOffset", lastMeta.get("nextOffset")); - ret.put("partition", partition.partition); - ret.put("broker", ImmutableMap.of("host", partition.host.host, "port", partition.host.port)); - ret.put("topic", _config.topic); - ret.put("topology", ImmutableMap.of("name", _topologyName, "id", _topologyInstanceId)); - return ret; - } - } - } - - private Map doEmitNewPartitionBatch(SimpleConsumer consumer, Partition partition, TridentCollector collector, Map lastMeta) { - long offset; - if (lastMeta != null) { - String lastInstanceId = null; - Map lastTopoMeta = (Map) lastMeta.get("topology"); - if (lastTopoMeta != null) { - lastInstanceId = (String) lastTopoMeta.get("id"); - } - if (_config.forceFromStart && !_topologyInstanceId.equals(lastInstanceId)) { - offset = KafkaUtils.getOffset(consumer, _config.topic, partition.partition, _config.startOffsetTime); - } else { - offset = (Long) lastMeta.get("nextOffset"); - } - } else { - long startTime = kafka.api.OffsetRequest.LatestTime(); - if (_config.forceFromStart) startTime = _config.startOffsetTime; - offset = KafkaUtils.getOffset(consumer, _config.topic, partition.partition, startTime); - } - ByteBufferMessageSet msgs; - try { - msgs = fetchMessages(consumer, partition, offset); - } catch (Exception e) { - if (e instanceof ConnectException) { - throw new FailedFetchException(e); - } else { - throw new RuntimeException(e); - } - } - long endoffset = offset; - for (MessageAndOffset msg : msgs) { - emit(collector, msg.message()); - endoffset = msg.nextOffset(); - } - Map newMeta = new HashMap(); - newMeta.put("offset", offset); - newMeta.put("nextOffset", endoffset); - newMeta.put("instanceId", _topologyInstanceId); - newMeta.put("partition", partition.partition); - newMeta.put("broker", ImmutableMap.of("host", partition.host.host, "port", partition.host.port)); - newMeta.put("topic", _config.topic); - newMeta.put("topology", ImmutableMap.of("name", _topologyName, "id", _topologyInstanceId)); - return newMeta; - } - - private ByteBufferMessageSet fetchMessages(SimpleConsumer consumer, Partition partition, long offset) { - ByteBufferMessageSet msgs; - long start = System.nanoTime(); - FetchRequestBuilder builder = new FetchRequestBuilder(); - FetchRequest fetchRequest = builder.addFetch(_config.topic, partition.partition, offset, _config.fetchSizeBytes).clientId(_config.clientId).build(); - msgs = consumer.fetch(fetchRequest).messageSet(_config.topic, partition.partition); - long end = System.nanoTime(); - long millis = (end - start) / 1000000; - _kafkaMeanFetchLatencyMetric.update(millis); - _kafkaMaxFetchLatencyMetric.update(millis); - return msgs; - } - - /** - * re-emit the batch described by the meta data provided - * - * @param attempt - * @param collector - * @param partition - * @param meta - */ - private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map meta) { - LOG.info("re-emitting batch, attempt " + attempt); - String instanceId = (String) meta.get("instanceId"); - if (!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) { - SimpleConsumer consumer = _connections.register(partition); - long offset = (Long) meta.get("offset"); - long nextOffset = (Long) meta.get("nextOffset"); - ByteBufferMessageSet msgs = fetchMessages(consumer, partition, offset); - for (MessageAndOffset msg : msgs) { - if (offset == nextOffset) break; - if (offset > nextOffset) { - throw new RuntimeException("Error when re-emitting batch. overshot the end offset"); - } - emit(collector, msg.message()); - offset = msg.nextOffset(); - } - } - } - - private void emit(TridentCollector collector, Message msg) { - Iterable> values = - _config.scheme.deserialize(Utils.toByteArray(msg.payload())); - if (values != null) { - for (List value : values) { - collector.emit(value); - } - } - } - - private void clear() { - _connections.clear(); - } - - private List orderPartitions(GlobalPartitionInformation partitions) { - return partitions.getOrderedPartitions(); - } - - private void refresh(List list) { - _connections.clear(); - _kafkaOffsetMetric.refreshPartitions(new HashSet(list)); - } - - - public IOpaquePartitionedTridentSpout.Emitter asOpaqueEmitter() { - - return new IOpaquePartitionedTridentSpout.Emitter() { - - /** - * Emit a batch of tuples for a partition/transaction. - * - * Return the metadata describing this batch that will be used as lastPartitionMeta - * for defining the parameters of the next batch. - */ - @Override - public Map emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, Partition partition, Map map) { - return emitNewPartitionBatch(transactionAttempt, tridentCollector, partition, map); - } - - @Override - public void refreshPartitions(List partitions) { - refresh(partitions); - } - - @Override - public List getOrderedPartitions(GlobalPartitionInformation partitionInformation) { - return orderPartitions(partitionInformation); - } - - @Override - public void close() { - clear(); - } - }; - } - - public IPartitionedTridentSpout.Emitter asTransactionalEmitter() { - return new IPartitionedTridentSpout.Emitter() { - - /** - * Emit a batch of tuples for a partition/transaction that's never been emitted before. - * Return the metadata that can be used to reconstruct this partition/batch in the future. - */ - @Override - public Map emitPartitionBatchNew(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, Partition partition, Map map) { - return failFastEmitNewPartitionBatch(transactionAttempt, tridentCollector, partition, map); - } - - /** - * Emit a batch of tuples for a partition/transaction that has been emitted before, using - * the metadata created when it was first emitted. - */ - @Override - public void emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, Partition partition, Map map) { - reEmitPartitionBatch(transactionAttempt, tridentCollector, partition, map); - } - - /** - * This method is called when this task is responsible for a new set of partitions. Should be used - * to manage things like connections to brokers. - */ - @Override - public void refreshPartitions(List partitions) { - refresh(partitions); - } - - @Override - public List getOrderedPartitions(GlobalPartitionInformation partitionInformation) { - return orderPartitions(partitionInformation); - } - - @Override - public void close() { - clear(); - } - }; - - } + public static final Logger LOG = LoggerFactory.getLogger(TridentKafkaEmitter.class); + + private DynamicPartitionConnections _connections; + private String _topologyName; + private KafkaUtils.KafkaOffsetMetric _kafkaOffsetMetric; + private ReducedMetric _kafkaMeanFetchLatencyMetric; + private CombinedMetric _kafkaMaxFetchLatencyMetric; + private TridentKafkaConfig _config; + private String _topologyInstanceId; + + public TridentKafkaEmitter(Map conf, TopologyContext context, TridentKafkaConfig config, String topologyInstanceId) { + _config = config; + _topologyInstanceId = topologyInstanceId; + _connections = new DynamicPartitionConnections(_config, KafkaUtils.makeBrokerReader(conf, _config)); + _topologyName = (String) conf.get(Config.TOPOLOGY_NAME); + _kafkaOffsetMetric = new KafkaUtils.KafkaOffsetMetric(_config.topic, _connections); + context.registerMetric("kafkaOffset", _kafkaOffsetMetric, 60); + _kafkaMeanFetchLatencyMetric = context.registerMetric("kafkaFetchAvg", new MeanReducer(), 60); + _kafkaMaxFetchLatencyMetric = context.registerMetric("kafkaFetchMax", new MaxMetric(), 60); + } + + + private Map failFastEmitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map lastMeta) { + SimpleConsumer consumer = _connections.register(partition); + Map ret = doEmitNewPartitionBatch(consumer, partition, collector, lastMeta); + _kafkaOffsetMetric.setLatestEmittedOffset(partition, (Long) ret.get("offset")); + return ret; + } + + private Map emitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map lastMeta) { + try { + return failFastEmitNewPartitionBatch(attempt, collector, partition, lastMeta); + } catch (FailedFetchException e) { + LOG.warn("Failed to fetch from partition " + partition); + if (lastMeta == null) { + return null; + } else { + Map ret = new HashMap(); + ret.put("offset", lastMeta.get("nextOffset")); + ret.put("nextOffset", lastMeta.get("nextOffset")); + ret.put("partition", partition.partition); + ret.put("broker", ImmutableMap.of("host", partition.host.host, "port", partition.host.port)); + ret.put("topic", _config.topic); + ret.put("topology", ImmutableMap.of("name", _topologyName, "id", _topologyInstanceId)); + return ret; + } + } + } + + private Map doEmitNewPartitionBatch(SimpleConsumer consumer, Partition partition, TridentCollector collector, Map lastMeta) { + long offset; + if (lastMeta != null) { + String lastInstanceId = null; + Map lastTopoMeta = (Map) lastMeta.get("topology"); + if (lastTopoMeta != null) { + lastInstanceId = (String) lastTopoMeta.get("id"); + } + if (_config.forceFromStart && !_topologyInstanceId.equals(lastInstanceId)) { + offset = KafkaUtils.getOffset(consumer, _config.topic, partition.partition, _config.startOffsetTime); + } else { + offset = (Long) lastMeta.get("nextOffset"); + } + } else { + long startTime = kafka.api.OffsetRequest.LatestTime(); + if (_config.forceFromStart) { + startTime = _config.startOffsetTime; + } + offset = KafkaUtils.getOffset(consumer, _config.topic, partition.partition, startTime); + } + ByteBufferMessageSet msgs; + try { + msgs = fetchMessages(consumer, partition, offset); + } catch (Exception e) { + if (e instanceof ConnectException) { + throw new FailedFetchException(e); + } else { + throw new RuntimeException(e); + } + } + long endoffset = offset; + for (MessageAndOffset msg : msgs) { + emit(collector, msg.message()); + endoffset = msg.nextOffset(); + } + Map newMeta = new HashMap(); + newMeta.put("offset", offset); + newMeta.put("nextOffset", endoffset); + newMeta.put("instanceId", _topologyInstanceId); + newMeta.put("partition", partition.partition); + newMeta.put("broker", ImmutableMap.of("host", partition.host.host, "port", partition.host.port)); + newMeta.put("topic", _config.topic); + newMeta.put("topology", ImmutableMap.of("name", _topologyName, "id", _topologyInstanceId)); + return newMeta; + } + + private ByteBufferMessageSet fetchMessages(SimpleConsumer consumer, Partition partition, long offset) { + ByteBufferMessageSet msgs; + long start = System.nanoTime(); + FetchRequestBuilder builder = new FetchRequestBuilder(); + FetchRequest fetchRequest = builder.addFetch(_config.topic, partition.partition, offset, _config.fetchSizeBytes).clientId(_config.clientId).build(); + msgs = consumer.fetch(fetchRequest).messageSet(_config.topic, partition.partition); + long end = System.nanoTime(); + long millis = (end - start) / 1000000; + _kafkaMeanFetchLatencyMetric.update(millis); + _kafkaMaxFetchLatencyMetric.update(millis); + return msgs; + } + + /** + * re-emit the batch described by the meta data provided + * + * @param attempt + * @param collector + * @param partition + * @param meta + */ + private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map meta) { + LOG.info("re-emitting batch, attempt " + attempt); + String instanceId = (String) meta.get("instanceId"); + if (!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) { + SimpleConsumer consumer = _connections.register(partition); + long offset = (Long) meta.get("offset"); + long nextOffset = (Long) meta.get("nextOffset"); + ByteBufferMessageSet msgs = fetchMessages(consumer, partition, offset); + for (MessageAndOffset msg : msgs) { + if (offset == nextOffset) { + break; + } + if (offset > nextOffset) { + throw new RuntimeException("Error when re-emitting batch. overshot the end offset"); + } + emit(collector, msg.message()); + offset = msg.nextOffset(); + } + } + } + + private void emit(TridentCollector collector, Message msg) { + Iterable> values = + _config.scheme.deserialize(Utils.toByteArray(msg.payload())); + if (values != null) { + for (List value : values) { + collector.emit(value); + } + } + } + + private void clear() { + _connections.clear(); + } + + private List orderPartitions(GlobalPartitionInformation partitions) { + return partitions.getOrderedPartitions(); + } + + private void refresh(List list) { + _connections.clear(); + _kafkaOffsetMetric.refreshPartitions(new HashSet(list)); + } + + + public IOpaquePartitionedTridentSpout.Emitter asOpaqueEmitter() { + + return new IOpaquePartitionedTridentSpout.Emitter() { + + /** + * Emit a batch of tuples for a partition/transaction. + * + * Return the metadata describing this batch that will be used as lastPartitionMeta + * for defining the parameters of the next batch. + */ + @Override + public Map emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, Partition partition, Map map) { + return emitNewPartitionBatch(transactionAttempt, tridentCollector, partition, map); + } + + @Override + public void refreshPartitions(List partitions) { + refresh(partitions); + } + + @Override + public List getOrderedPartitions(GlobalPartitionInformation partitionInformation) { + return orderPartitions(partitionInformation); + } + + @Override + public void close() { + clear(); + } + }; + } + + public IPartitionedTridentSpout.Emitter asTransactionalEmitter() { + return new IPartitionedTridentSpout.Emitter() { + + /** + * Emit a batch of tuples for a partition/transaction that's never been emitted before. + * Return the metadata that can be used to reconstruct this partition/batch in the future. + */ + @Override + public Map emitPartitionBatchNew(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, Partition partition, Map map) { + return failFastEmitNewPartitionBatch(transactionAttempt, tridentCollector, partition, map); + } + + /** + * Emit a batch of tuples for a partition/transaction that has been emitted before, using + * the metadata created when it was first emitted. + */ + @Override + public void emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, Partition partition, Map map) { + reEmitPartitionBatch(transactionAttempt, tridentCollector, partition, map); + } + + /** + * This method is called when this task is responsible for a new set of partitions. Should be used + * to manage things like connections to brokers. + */ + @Override + public void refreshPartitions(List partitions) { + refresh(partitions); + } + + @Override + public List getOrderedPartitions(GlobalPartitionInformation partitionInformation) { + return orderPartitions(partitionInformation); + } + + @Override + public void close() { + clear(); + } + }; + + } } diff --git a/src/jvm/storm/kafka/trident/ZkBrokerReader.java b/src/jvm/storm/kafka/trident/ZkBrokerReader.java index 0a53137967..5e2361d13c 100644 --- a/src/jvm/storm/kafka/trident/ZkBrokerReader.java +++ b/src/jvm/storm/kafka/trident/ZkBrokerReader.java @@ -10,9 +10,9 @@ public class ZkBrokerReader implements IBrokerReader { - public static final Logger LOG = LoggerFactory.getLogger(ZkBrokerReader.class); + public static final Logger LOG = LoggerFactory.getLogger(ZkBrokerReader.class); - GlobalPartitionInformation cachedBrokers; + GlobalPartitionInformation cachedBrokers; DynamicBrokersReader reader; long lastRefreshTimeMs; @@ -30,16 +30,16 @@ public ZkBrokerReader(Map conf, String topic, ZkHosts hosts) { @Override public GlobalPartitionInformation getCurrentBrokers() { long currTime = System.currentTimeMillis(); - if(currTime > lastRefreshTimeMs + refreshMillis) { - LOG.info("brokers need refreshing because " + refreshMillis + "ms have expired"); + if (currTime > lastRefreshTimeMs + refreshMillis) { + LOG.info("brokers need refreshing because " + refreshMillis + "ms have expired"); cachedBrokers = reader.getBrokerInfo(); lastRefreshTimeMs = currTime; - } + } return cachedBrokers; } @Override public void close() { reader.close(); - } + } } diff --git a/src/test/storm/kafka/DynamicBrokersReaderTest.java b/src/test/storm/kafka/DynamicBrokersReaderTest.java index a6c2309b08..fd90c3c777 100644 --- a/src/test/storm/kafka/DynamicBrokersReaderTest.java +++ b/src/test/storm/kafka/DynamicBrokersReaderTest.java @@ -21,133 +21,133 @@ * Time: 20:35 */ public class DynamicBrokersReaderTest { - private DynamicBrokersReader dynamicBrokersReader; - private String masterPath = "/brokers"; - private String topic = "testing"; - private CuratorFramework zookeeper; - private TestingServer server; - - @Before - public void setUp() throws Exception { - server = new TestingServer(); - String connectionString = server.getConnectString(); - Map conf = new HashMap(); - conf.put(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT, 1000); - conf.put(Config.STORM_ZOOKEEPER_RETRY_TIMES, 4); - conf.put(Config.STORM_ZOOKEEPER_RETRY_INTERVAL, 5); - ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3); - zookeeper = CuratorFrameworkFactory.newClient(connectionString, retryPolicy); - dynamicBrokersReader = new DynamicBrokersReader(conf, connectionString, masterPath, topic); - zookeeper.start(); - } - - @After - public void tearDown() throws Exception { - server.close(); - } - - private void addPartition(int id, String host, int port) throws Exception { - writePartitionId(id); - writeLeader(id, 0); - writeLeaderDetails(0, host, port); - } - - private void addPartition(int id, int leader, String host, int port) throws Exception { - writePartitionId(id); - writeLeader(id, leader); - writeLeaderDetails(leader, host, port); - } - - private void writePartitionId(int id) throws Exception { - String path = dynamicBrokersReader.partitionPath(); - writeDataToPath(path, ("" + id)); - } - - private void writeDataToPath(String path, String data) throws Exception { - ZKPaths.mkdirs(zookeeper.getZookeeperClient().getZooKeeper(), path); - zookeeper.setData().forPath(path, data.getBytes()); - } - - private void writeLeader(int id, int leaderId) throws Exception { - String path = dynamicBrokersReader.partitionPath() + "/" + id + "/state"; - String value = " { \"controller_epoch\":4, \"isr\":[ 1, 0 ], \"leader\":" +leaderId + ", \"leader_epoch\":1, \"version\":1 }"; - writeDataToPath(path, value); - } - - private void writeLeaderDetails(int leaderId, String host, int port) throws Exception{ - String path = dynamicBrokersReader.brokerPath() + "/" + leaderId; - String value = "{ \"host\":\"" + host + "\", \"jmx_port\":9999, \"port\":" + port + ", \"version\":1 }"; - writeDataToPath(path, value); - } - - @Test - public void testGetBrokerInfo() throws Exception { - String host = "localhost"; - int port = 9092; - int partition = 0; - addPartition(partition, host, port); - GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); - assertEquals(1,brokerInfo.getOrderedPartitions().size()); - assertEquals(port, brokerInfo.getHostFor(partition).port); - assertEquals(host, brokerInfo.getHostFor(partition).host); - } - - - @Test - public void testMultiplePartitionsOnDifferentHosts() throws Exception { - String host = "localhost"; - int port = 9092; - int secondPort = 9093; - int partition = 0; - int secondPartition = partition + 1; - addPartition(partition, 0, host, port); - addPartition(secondPartition, 1, host, secondPort); - - GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); - assertEquals(2,brokerInfo.getOrderedPartitions().size()); - - assertEquals(port, brokerInfo.getHostFor(partition).port); - assertEquals(host, brokerInfo.getHostFor(partition).host); - - assertEquals(secondPort, brokerInfo.getHostFor(secondPartition).port); - assertEquals(host, brokerInfo.getHostFor(secondPartition).host); - } - - - @Test - public void testMultiplePartitionsOnSameHost() throws Exception { - String host = "localhost"; - int port = 9092; - int partition = 0; - int secondPartition = partition + 1; - addPartition(partition, 0, host, port); - addPartition(secondPartition, 0, host, port); - - GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); - assertEquals(2,brokerInfo.getOrderedPartitions().size()); - - assertEquals(port, brokerInfo.getHostFor(partition).port); - assertEquals(host, brokerInfo.getHostFor(partition).host); - - assertEquals(port, brokerInfo.getHostFor(secondPartition).port); - assertEquals(host, brokerInfo.getHostFor(secondPartition).host); - } - - @Test - public void testSwitchHostForPartition() throws Exception { - String host = "localhost"; - int port = 9092; - int partition = 0; - addPartition(partition, host, port); - GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); - assertEquals(port, brokerInfo.getHostFor(partition).port); - assertEquals(host, brokerInfo.getHostFor(partition).host); - - String newHost = host + "switch"; - int newPort = port + 1; - addPartition(partition, newHost, newPort); - brokerInfo = dynamicBrokersReader.getBrokerInfo(); - assertEquals(newPort, brokerInfo.getHostFor(partition).port); - assertEquals(newHost, brokerInfo.getHostFor(partition).host); - } + private DynamicBrokersReader dynamicBrokersReader; + private String masterPath = "/brokers"; + private String topic = "testing"; + private CuratorFramework zookeeper; + private TestingServer server; + + @Before + public void setUp() throws Exception { + server = new TestingServer(); + String connectionString = server.getConnectString(); + Map conf = new HashMap(); + conf.put(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT, 1000); + conf.put(Config.STORM_ZOOKEEPER_RETRY_TIMES, 4); + conf.put(Config.STORM_ZOOKEEPER_RETRY_INTERVAL, 5); + ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3); + zookeeper = CuratorFrameworkFactory.newClient(connectionString, retryPolicy); + dynamicBrokersReader = new DynamicBrokersReader(conf, connectionString, masterPath, topic); + zookeeper.start(); + } + + @After + public void tearDown() throws Exception { + server.close(); + } + + private void addPartition(int id, String host, int port) throws Exception { + writePartitionId(id); + writeLeader(id, 0); + writeLeaderDetails(0, host, port); + } + + private void addPartition(int id, int leader, String host, int port) throws Exception { + writePartitionId(id); + writeLeader(id, leader); + writeLeaderDetails(leader, host, port); + } + + private void writePartitionId(int id) throws Exception { + String path = dynamicBrokersReader.partitionPath(); + writeDataToPath(path, ("" + id)); + } + + private void writeDataToPath(String path, String data) throws Exception { + ZKPaths.mkdirs(zookeeper.getZookeeperClient().getZooKeeper(), path); + zookeeper.setData().forPath(path, data.getBytes()); + } + + private void writeLeader(int id, int leaderId) throws Exception { + String path = dynamicBrokersReader.partitionPath() + "/" + id + "/state"; + String value = " { \"controller_epoch\":4, \"isr\":[ 1, 0 ], \"leader\":" + leaderId + ", \"leader_epoch\":1, \"version\":1 }"; + writeDataToPath(path, value); + } + + private void writeLeaderDetails(int leaderId, String host, int port) throws Exception { + String path = dynamicBrokersReader.brokerPath() + "/" + leaderId; + String value = "{ \"host\":\"" + host + "\", \"jmx_port\":9999, \"port\":" + port + ", \"version\":1 }"; + writeDataToPath(path, value); + } + + @Test + public void testGetBrokerInfo() throws Exception { + String host = "localhost"; + int port = 9092; + int partition = 0; + addPartition(partition, host, port); + GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); + assertEquals(1, brokerInfo.getOrderedPartitions().size()); + assertEquals(port, brokerInfo.getHostFor(partition).port); + assertEquals(host, brokerInfo.getHostFor(partition).host); + } + + + @Test + public void testMultiplePartitionsOnDifferentHosts() throws Exception { + String host = "localhost"; + int port = 9092; + int secondPort = 9093; + int partition = 0; + int secondPartition = partition + 1; + addPartition(partition, 0, host, port); + addPartition(secondPartition, 1, host, secondPort); + + GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); + assertEquals(2, brokerInfo.getOrderedPartitions().size()); + + assertEquals(port, brokerInfo.getHostFor(partition).port); + assertEquals(host, brokerInfo.getHostFor(partition).host); + + assertEquals(secondPort, brokerInfo.getHostFor(secondPartition).port); + assertEquals(host, brokerInfo.getHostFor(secondPartition).host); + } + + + @Test + public void testMultiplePartitionsOnSameHost() throws Exception { + String host = "localhost"; + int port = 9092; + int partition = 0; + int secondPartition = partition + 1; + addPartition(partition, 0, host, port); + addPartition(secondPartition, 0, host, port); + + GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); + assertEquals(2, brokerInfo.getOrderedPartitions().size()); + + assertEquals(port, brokerInfo.getHostFor(partition).port); + assertEquals(host, brokerInfo.getHostFor(partition).host); + + assertEquals(port, brokerInfo.getHostFor(secondPartition).port); + assertEquals(host, brokerInfo.getHostFor(secondPartition).host); + } + + @Test + public void testSwitchHostForPartition() throws Exception { + String host = "localhost"; + int port = 9092; + int partition = 0; + addPartition(partition, host, port); + GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); + assertEquals(port, brokerInfo.getHostFor(partition).port); + assertEquals(host, brokerInfo.getHostFor(partition).host); + + String newHost = host + "switch"; + int newPort = port + 1; + addPartition(partition, newHost, newPort); + brokerInfo = dynamicBrokersReader.getBrokerInfo(); + assertEquals(newPort, brokerInfo.getHostFor(partition).port); + assertEquals(newHost, brokerInfo.getHostFor(partition).host); + } } From da18bd858035c495edb41cc018fe706a3f7ef831 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 22 Dec 2013 16:10:14 +0000 Subject: [PATCH 245/349] removed empty statement --- src/jvm/storm/kafka/trident/KafkaUtils.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/jvm/storm/kafka/trident/KafkaUtils.java b/src/jvm/storm/kafka/trident/KafkaUtils.java index 18dd851fb2..e4ba3b35de 100644 --- a/src/jvm/storm/kafka/trident/KafkaUtils.java +++ b/src/jvm/storm/kafka/trident/KafkaUtils.java @@ -109,6 +109,4 @@ public void refreshPartitions(Set partitions) { } } } - - ; } From 735b87f78459ec686017c538ef50d95a7db9584b Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Wed, 25 Dec 2013 16:28:16 +0000 Subject: [PATCH 246/349] Renamed HostPort to Broker --- .../kafka/{HostPort.java => Broker.java} | 22 +++++++-------- src/jvm/storm/kafka/DynamicBrokersReader.java | 8 +++--- .../kafka/DynamicPartitionConnections.java | 10 +++---- src/jvm/storm/kafka/Partition.java | 4 +-- .../kafka/StaticPartitionConnections.java | 2 +- .../trident/GlobalPartitionInformation.java | 17 +++++------ .../storm/kafka/DynamicBrokersReaderTest.java | 28 +++++++++---------- 7 files changed, 46 insertions(+), 45 deletions(-) rename src/jvm/storm/kafka/{HostPort.java => Broker.java} (65%) diff --git a/src/jvm/storm/kafka/HostPort.java b/src/jvm/storm/kafka/Broker.java similarity index 65% rename from src/jvm/storm/kafka/HostPort.java rename to src/jvm/storm/kafka/Broker.java index 53698584ec..66e6112a89 100644 --- a/src/jvm/storm/kafka/HostPort.java +++ b/src/jvm/storm/kafka/Broker.java @@ -2,22 +2,22 @@ import java.io.Serializable; -public class HostPort implements Serializable, Comparable { - public String host; - public int port; +public class Broker implements Serializable, Comparable { + public final String host; + public final int port; - public HostPort(String host, int port) { + public Broker(String host, int port) { this.host = host; this.port = port; } - public HostPort(String host) { + public Broker(String host) { this(host, 9092); } @Override public boolean equals(Object o) { - HostPort other = (HostPort) o; + Broker other = (Broker) o; return host.equals(other.host) && port == other.port; } @@ -31,13 +31,13 @@ public String toString() { return host + ":" + port; } - public static HostPort fromString(String host) { - HostPort hp; + public static Broker fromString(String host) { + Broker hp; String[] spec = host.split(":"); if (spec.length == 1) { - hp = new HostPort(spec[0]); + hp = new Broker(spec[0]); } else if (spec.length == 2) { - hp = new HostPort(spec[0], Integer.parseInt(spec[1])); + hp = new Broker(spec[0], Integer.parseInt(spec[1])); } else { throw new IllegalArgumentException("Invalid host specification: " + host); } @@ -46,7 +46,7 @@ public static HostPort fromString(String host) { @Override - public int compareTo(HostPort o) { + public int compareTo(Broker o) { if (this.host.equals(o.host)) { return this.port - o.port; } else { diff --git a/src/jvm/storm/kafka/DynamicBrokersReader.java b/src/jvm/storm/kafka/DynamicBrokersReader.java index c802baf3ea..5b1d750bab 100644 --- a/src/jvm/storm/kafka/DynamicBrokersReader.java +++ b/src/jvm/storm/kafka/DynamicBrokersReader.java @@ -51,8 +51,8 @@ public GlobalPartitionInformation getBrokerInfo() { int leader = getLeaderFor(partition); String path = brokerInfoPath + "/" + leader; try { - byte[] hostPortData = _curator.getData().forPath(path); - HostPort hp = getBrokerHost(hostPortData); + byte[] brokerData = _curator.getData().forPath(path); + Broker hp = getBrokerHost(brokerData); globalPartitionInformation.addPartition(partition, hp); } catch (org.apache.zookeeper.KeeperException.NoNodeException e) { LOG.error("Node {} does not exist ", path); @@ -114,12 +114,12 @@ public void close() { * @param contents * @return */ - private HostPort getBrokerHost(byte[] contents) { + private Broker getBrokerHost(byte[] contents) { try { Map value = (Map) JSONValue.parse(new String(contents, "UTF-8")); String host = (String) value.get("host"); Integer port = ((Long) value.get("port")).intValue(); - return new HostPort(host, port); + return new Broker(host, port); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } diff --git a/src/jvm/storm/kafka/DynamicPartitionConnections.java b/src/jvm/storm/kafka/DynamicPartitionConnections.java index 7a799a0f1d..8d0115b68d 100644 --- a/src/jvm/storm/kafka/DynamicPartitionConnections.java +++ b/src/jvm/storm/kafka/DynamicPartitionConnections.java @@ -24,7 +24,7 @@ public ConnectionInfo(SimpleConsumer consumer) { } } - Map _connections = new HashMap(); + Map _connections = new HashMap(); KafkaConfig _config; IBrokerReader _reader; @@ -34,11 +34,11 @@ public DynamicPartitionConnections(KafkaConfig config, IBrokerReader brokerReade } public SimpleConsumer register(Partition partition) { - HostPort hostPort = _reader.getCurrentBrokers().getHostFor(partition.partition); - return register(hostPort, partition.partition); + Broker broker = _reader.getCurrentBrokers().getBrokerFor(partition.partition); + return register(broker, partition.partition); } - public SimpleConsumer register(HostPort host, int partition) { + public SimpleConsumer register(Broker host, int partition) { if (!_connections.containsKey(host)) { _connections.put(host, new ConnectionInfo(new SimpleConsumer(host.host, host.port, _config.socketTimeoutMs, _config.bufferSizeBytes, _config.clientId))); } @@ -55,7 +55,7 @@ public SimpleConsumer getConnection(Partition partition) { return null; } - public void unregister(HostPort port, int partition) { + public void unregister(Broker port, int partition) { ConnectionInfo info = _connections.get(port); info.partitions.remove(partition); if (info.partitions.isEmpty()) { diff --git a/src/jvm/storm/kafka/Partition.java b/src/jvm/storm/kafka/Partition.java index 87ab7b8128..bbb4fbbfb1 100644 --- a/src/jvm/storm/kafka/Partition.java +++ b/src/jvm/storm/kafka/Partition.java @@ -6,10 +6,10 @@ public class Partition implements ISpoutPartition { - public final HostPort host; + public final Broker host; public final int partition; - public Partition(HostPort host, int partition) { + public Partition(Broker host, int partition) { this.host = host; this.partition = partition; } diff --git a/src/jvm/storm/kafka/StaticPartitionConnections.java b/src/jvm/storm/kafka/StaticPartitionConnections.java index 4294362f9f..a9b9db1100 100644 --- a/src/jvm/storm/kafka/StaticPartitionConnections.java +++ b/src/jvm/storm/kafka/StaticPartitionConnections.java @@ -20,7 +20,7 @@ public StaticPartitionConnections(KafkaConfig conf) { public SimpleConsumer getConsumer(int partition) { if (!_kafka.containsKey(partition)) { - HostPort hp = hosts.getPartitionInformation().getHostFor(partition); + Broker hp = hosts.getPartitionInformation().getBrokerFor(partition); _kafka.put(partition, new SimpleConsumer(hp.host, hp.port, _config.socketTimeoutMs, _config.bufferSizeBytes, _config.clientId)); } diff --git a/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java b/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java index 6b0fdec90a..a7900092f2 100644 --- a/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java +++ b/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java @@ -1,6 +1,7 @@ package storm.kafka.trident; -import storm.kafka.HostPort; +import storm.kafka.Broker; +import storm.kafka.Broker; import storm.kafka.Partition; import java.io.Serializable; @@ -12,13 +13,13 @@ */ public class GlobalPartitionInformation implements Iterable, Serializable { - private Map partitionMap; + private Map partitionMap; public GlobalPartitionInformation() { - partitionMap = new TreeMap(); + partitionMap = new TreeMap(); } - public void addPartition(int partitionId, HostPort broker) { + public void addPartition(int partitionId, Broker broker) { partitionMap.put(partitionId, broker); } @@ -29,13 +30,13 @@ public String toString() { '}'; } - public HostPort getHostFor(Integer partitionId) { + public Broker getBrokerFor(Integer partitionId) { return partitionMap.get(partitionId); } public List getOrderedPartitions() { List partitions = new LinkedList(); - for (Map.Entry partition : partitionMap.entrySet()) { + for (Map.Entry partition : partitionMap.entrySet()) { partitions.add(new Partition(partition.getValue(), partition.getKey())); } return partitions; @@ -43,7 +44,7 @@ public List getOrderedPartitions() { @Override public Iterator iterator() { - final Iterator> iterator = partitionMap.entrySet().iterator(); + final Iterator> iterator = partitionMap.entrySet().iterator(); return new Iterator() { @Override @@ -53,7 +54,7 @@ public boolean hasNext() { @Override public Partition next() { - Map.Entry next = iterator.next(); + Map.Entry next = iterator.next(); return new Partition(next.getValue(), next.getKey()); } diff --git a/src/test/storm/kafka/DynamicBrokersReaderTest.java b/src/test/storm/kafka/DynamicBrokersReaderTest.java index fd90c3c777..47387e35eb 100644 --- a/src/test/storm/kafka/DynamicBrokersReaderTest.java +++ b/src/test/storm/kafka/DynamicBrokersReaderTest.java @@ -88,8 +88,8 @@ public void testGetBrokerInfo() throws Exception { addPartition(partition, host, port); GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); assertEquals(1, brokerInfo.getOrderedPartitions().size()); - assertEquals(port, brokerInfo.getHostFor(partition).port); - assertEquals(host, brokerInfo.getHostFor(partition).host); + assertEquals(port, brokerInfo.getBrokerFor(partition).port); + assertEquals(host, brokerInfo.getBrokerFor(partition).host); } @@ -106,11 +106,11 @@ public void testMultiplePartitionsOnDifferentHosts() throws Exception { GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); assertEquals(2, brokerInfo.getOrderedPartitions().size()); - assertEquals(port, brokerInfo.getHostFor(partition).port); - assertEquals(host, brokerInfo.getHostFor(partition).host); + assertEquals(port, brokerInfo.getBrokerFor(partition).port); + assertEquals(host, brokerInfo.getBrokerFor(partition).host); - assertEquals(secondPort, brokerInfo.getHostFor(secondPartition).port); - assertEquals(host, brokerInfo.getHostFor(secondPartition).host); + assertEquals(secondPort, brokerInfo.getBrokerFor(secondPartition).port); + assertEquals(host, brokerInfo.getBrokerFor(secondPartition).host); } @@ -126,11 +126,11 @@ public void testMultiplePartitionsOnSameHost() throws Exception { GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); assertEquals(2, brokerInfo.getOrderedPartitions().size()); - assertEquals(port, brokerInfo.getHostFor(partition).port); - assertEquals(host, brokerInfo.getHostFor(partition).host); + assertEquals(port, brokerInfo.getBrokerFor(partition).port); + assertEquals(host, brokerInfo.getBrokerFor(partition).host); - assertEquals(port, brokerInfo.getHostFor(secondPartition).port); - assertEquals(host, brokerInfo.getHostFor(secondPartition).host); + assertEquals(port, brokerInfo.getBrokerFor(secondPartition).port); + assertEquals(host, brokerInfo.getBrokerFor(secondPartition).host); } @Test @@ -140,14 +140,14 @@ public void testSwitchHostForPartition() throws Exception { int partition = 0; addPartition(partition, host, port); GlobalPartitionInformation brokerInfo = dynamicBrokersReader.getBrokerInfo(); - assertEquals(port, brokerInfo.getHostFor(partition).port); - assertEquals(host, brokerInfo.getHostFor(partition).host); + assertEquals(port, brokerInfo.getBrokerFor(partition).port); + assertEquals(host, brokerInfo.getBrokerFor(partition).host); String newHost = host + "switch"; int newPort = port + 1; addPartition(partition, newHost, newPort); brokerInfo = dynamicBrokersReader.getBrokerInfo(); - assertEquals(newPort, brokerInfo.getHostFor(partition).port); - assertEquals(newHost, brokerInfo.getHostFor(partition).host); + assertEquals(newPort, brokerInfo.getBrokerFor(partition).port); + assertEquals(newHost, brokerInfo.getBrokerFor(partition).host); } } From b5de86ed7f966018454fba667a42010f0f73f490 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 5 Jan 2014 12:17:55 +0000 Subject: [PATCH 247/349] Use partition id in zk path * added topology id to logs --- src/jvm/storm/kafka/Partition.java | 2 +- src/jvm/storm/kafka/PartitionManager.java | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/jvm/storm/kafka/Partition.java b/src/jvm/storm/kafka/Partition.java index bbb4fbbfb1..96a3ad7350 100644 --- a/src/jvm/storm/kafka/Partition.java +++ b/src/jvm/storm/kafka/Partition.java @@ -41,7 +41,7 @@ public String toString() { @Override public String getId() { - return toString(); + return "partition_" + partition; } } diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 623bc10fe1..e3e31dba6a 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -63,14 +63,16 @@ public PartitionManager(DynamicPartitionConnections connections, String topology String jsonTopologyId = null; Long jsonOffset = null; + String path = committedPath(); try { - Map json = _state.readJSON(committedPath()); + Map json = _state.readJSON(path); + LOG.info("Read partition information from: " + path + " --> " + json ); if (json != null) { jsonTopologyId = (String) ((Map) json.get("topology")).get("id"); jsonOffset = (Long) json.get("offset"); } } catch (Throwable e) { - LOG.warn("Error reading and/or parsing at ZkNode: " + committedPath(), e); + LOG.warn("Error reading and/or parsing at ZkNode: " + path, e); } if (!topologyInstanceId.equals(jsonTopologyId) && spoutConfig.forceFromStart) { @@ -81,7 +83,7 @@ public PartitionManager(DynamicPartitionConnections connections, String topology LOG.info("Setting last commit offset to HEAD."); } else { _committedTo = jsonOffset; - LOG.info("Read last commit offset from zookeeper: " + _committedTo); + LOG.info("Read last commit offset from zookeeper: " + _committedTo + "; old topology_id: " + jsonTopologyId + " - new topology_id: " + topologyInstanceId ); } LOG.info("Starting Kafka " + _consumer.host() + ":" + id.partition + " from offset " + _committedTo); @@ -205,11 +207,11 @@ public void commit() { LOG.info("Wrote committed offset to ZK: " + committedTo); _committedTo = committedTo; } - LOG.info("Committed offset " + committedTo + " for " + _partition); + LOG.info("Committed offset " + committedTo + " for " + _partition + " for topology: " + _topologyInstanceId); } private String committedPath() { - return _spoutConfig.zkRoot + "/" + _spoutConfig.id + "/" + _partition; + return _spoutConfig.zkRoot + "/" + _spoutConfig.id + "/" + _partition.getId(); } public long queryPartitionOffsetLatestTime() { From 4b6e9bcf90f52b3eef4817ff30cb8792b4c3b8e0 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 5 Jan 2014 12:25:22 +0000 Subject: [PATCH 248/349] added changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..2fb81fe190 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,2 @@ +## 0.3.0 +* updated partition path in zookeeper From 6b29f8f0bf56b473bf06e2fc7a0f822e5c87d9bc Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Tue, 7 Jan 2014 08:00:36 +0000 Subject: [PATCH 249/349] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index b734b5bffe..13b5b2c29b 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,7 @@ For information on how to use this library in your project see: [https://clojars.org/net.wurstmeister.storm/storm-kafka-0.8-plus](https://clojars.org/net.wurstmeister.storm/storm-kafka-0.8-plus) + +##Example Topologies: + +[https://github.com/wurstmeister/storm-kafka-0.8-plus-test](https://github.com/wurstmeister/storm-kafka-0.8-plus-test) From 95c60dbbbc80165969bae6cbbd1926207720e59c Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sat, 11 Jan 2014 16:34:01 +0000 Subject: [PATCH 250/349] Added error handling for fetch request * allow retry of fetch if offset was invalid * updated changelog --- CHANGELOG.md | 1 + .../{trident => }/FailedFetchException.java | 7 +- src/jvm/storm/kafka/KafkaConfig.java | 5 +- src/jvm/storm/kafka/KafkaError.java | 29 ++++++ src/jvm/storm/kafka/KafkaSpout.java | 24 ----- .../storm/kafka/{trident => }/KafkaUtils.java | 51 ++++++++++- src/jvm/storm/kafka/PartitionManager.java | 11 +-- src/jvm/storm/kafka/trident/Coordinator.java | 1 + .../kafka/trident/TridentKafkaEmitter.java | 21 +---- src/test/storm/kafka/KafkaTestBroker.java | 52 +++++++++++ src/test/storm/kafka/KafkaUtilsTest.java | 90 +++++++++++++++++++ 11 files changed, 234 insertions(+), 58 deletions(-) rename src/jvm/storm/kafka/{trident => }/FailedFetchException.java (56%) create mode 100644 src/jvm/storm/kafka/KafkaError.java rename src/jvm/storm/kafka/{trident => }/KafkaUtils.java (67%) create mode 100644 src/test/storm/kafka/KafkaTestBroker.java create mode 100644 src/test/storm/kafka/KafkaUtilsTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fb81fe190..ced0ffc975 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,2 +1,3 @@ ## 0.3.0 * updated partition path in zookeeper +* added error handling for fetch request diff --git a/src/jvm/storm/kafka/trident/FailedFetchException.java b/src/jvm/storm/kafka/FailedFetchException.java similarity index 56% rename from src/jvm/storm/kafka/trident/FailedFetchException.java rename to src/jvm/storm/kafka/FailedFetchException.java index c4fcc6181d..0bd1123ac7 100644 --- a/src/jvm/storm/kafka/trident/FailedFetchException.java +++ b/src/jvm/storm/kafka/FailedFetchException.java @@ -1,6 +1,11 @@ -package storm.kafka.trident; +package storm.kafka; public class FailedFetchException extends RuntimeException { + + public FailedFetchException(String message) { + super(message); + } + public FailedFetchException(Exception e) { super(e); } diff --git a/src/jvm/storm/kafka/KafkaConfig.java b/src/jvm/storm/kafka/KafkaConfig.java index e24197854a..dddcead1bd 100644 --- a/src/jvm/storm/kafka/KafkaConfig.java +++ b/src/jvm/storm/kafka/KafkaConfig.java @@ -17,6 +17,7 @@ public class KafkaConfig implements Serializable { public MultiScheme scheme = new RawMultiScheme(); public boolean forceFromStart = false; public long startOffsetTime = kafka.api.OffsetRequest.EarliestTime(); + public boolean useStartOffsetTimeIfOffsetOutOfRange = true; public KafkaConfig(BrokerHosts hosts, String topic) { this(hosts, topic, kafka.api.OffsetRequest.DefaultClientId()); @@ -28,8 +29,4 @@ public KafkaConfig(BrokerHosts hosts, String topic, String clientId) { this.clientId = clientId; } - public void forceStartOffsetTime(long millis) { - startOffsetTime = millis; - forceFromStart = true; - } } diff --git a/src/jvm/storm/kafka/KafkaError.java b/src/jvm/storm/kafka/KafkaError.java new file mode 100644 index 0000000000..260ab91568 --- /dev/null +++ b/src/jvm/storm/kafka/KafkaError.java @@ -0,0 +1,29 @@ +package storm.kafka; + +/** + * Date: 11/01/2014 + * Time: 14:21 + */ +public enum KafkaError { + NO_ERROR, + OFFSET_OUT_OF_RANGE, + INVALID_MESSAGE, + UNKNOWN_TOPIC_OR_PARTITION, + INVALID_FETCH_SIZE, + LEADER_NOT_AVAILABLE, + NOT_LEADER_FOR_PARTITION, + REQUEST_TIMED_OUT, + BROKER_NOT_AVAILABLE, + REPLICA_NOT_AVAILABLE, + MESSAGE_SIZE_TOO_LARGE, + STALE_CONTROLLER_EPOCH, + UNKNOWN; + + public static KafkaError getError(short errorCode) { + if (errorCode < 0) { + return UNKNOWN; + } else { + return values()[errorCode]; + } + } +} diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index cf407ad577..d0975106c8 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -10,7 +10,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import storm.kafka.PartitionManager.KafkaMessageId; -import storm.kafka.trident.KafkaUtils; import java.util.*; @@ -171,27 +170,4 @@ private void commit() { } } - public static void main(String[] args) { -// TopologyBuilder builder = new TopologyBuilder(); -// List hosts = new ArrayList(); -// hosts.add("localhost"); -// SpoutConfig spoutConf = SpoutConfig.fromHostStrings(hosts, 8, "clicks", "/kafkastorm", "id"); -// spoutConf.scheme = new SchemeAsMultiScheme(new StringScheme()); -// spoutConf.forceStartOffsetTime(-2); -// -// // spoutConf.zkServers = new ArrayList() {{ -// // add("localhost"); -// // }}; -// // spoutConf.zkPort = 2181; -// -// builder.setSpout("spout", new KafkaSpout(spoutConf), 3); -// -// Config conf = new Config(); -// //conf.setDebug(true); -// -// LocalCluster cluster = new LocalCluster(); -// cluster.submitTopology("kafka-test", conf, builder.createTopology()); -// -// Utils.sleep(600000); - } } diff --git a/src/jvm/storm/kafka/trident/KafkaUtils.java b/src/jvm/storm/kafka/KafkaUtils.java similarity index 67% rename from src/jvm/storm/kafka/trident/KafkaUtils.java rename to src/jvm/storm/kafka/KafkaUtils.java index e4ba3b35de..5094f14ee7 100644 --- a/src/jvm/storm/kafka/trident/KafkaUtils.java +++ b/src/jvm/storm/kafka/KafkaUtils.java @@ -1,20 +1,29 @@ -package storm.kafka.trident; +package storm.kafka; import backtype.storm.metric.api.IMetric; +import kafka.api.FetchRequest; +import kafka.api.FetchRequestBuilder; import kafka.api.PartitionOffsetRequestInfo; import kafka.common.TopicAndPartition; +import kafka.javaapi.FetchResponse; import kafka.javaapi.OffsetRequest; import kafka.javaapi.consumer.SimpleConsumer; +import kafka.javaapi.message.ByteBufferMessageSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import storm.kafka.*; +import storm.kafka.trident.IBrokerReader; +import storm.kafka.trident.StaticBrokerReader; +import storm.kafka.trident.ZkBrokerReader; +import java.net.ConnectException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Set; + public class KafkaUtils { + public static final Logger LOG = LoggerFactory.getLogger(KafkaUtils.class); private static final int NO_OFFSET = -5; @@ -109,4 +118,42 @@ public void refreshPartitions(Set partitions) { } } } + + public static ByteBufferMessageSet fetchMessages(KafkaConfig config, SimpleConsumer consumer, Partition partition, long offset) { + ByteBufferMessageSet msgs = null; + String topic = config.topic; + int partitionId = partition.partition; + for (int errors = 0; errors < 2 && msgs == null; errors++) { + FetchRequestBuilder builder = new FetchRequestBuilder(); + FetchRequest fetchRequest = builder.addFetch(topic, partitionId, offset, config.fetchSizeBytes). + clientId(config.clientId).build(); + FetchResponse fetchResponse; + try { + fetchResponse = consumer.fetch(fetchRequest); + } catch (Exception e) { + if (e instanceof ConnectException) { + throw new FailedFetchException(e); + } else { + throw new RuntimeException(e); + } + } + if (fetchResponse.hasError()) { + KafkaError error = KafkaError.getError(fetchResponse.errorCode(topic, partitionId)); + if (error.equals(KafkaError.OFFSET_OUT_OF_RANGE) && config.useStartOffsetTimeIfOffsetOutOfRange && errors == 0) { + long startOffset = getOffset(consumer, topic, partitionId, config.startOffsetTime); + LOG.warn("Got fetch request with offset out of range: [" + offset + "]; " + + "retrying with default start offset time from configuration. " + + "configured start offset time: [" + config.startOffsetTime + "] offset: [" + startOffset + "]"); + offset = startOffset; + } else { + String message = "Error fetching data from [" + partition + "] for topic [" + topic + "]: [" + error + "]"; + LOG.error(message); + throw new FailedFetchException(message); + } + } else { + msgs = fetchResponse.messageSet(topic, partitionId); + } + } + return msgs; + } } diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index e3e31dba6a..0861c255cd 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -8,7 +8,6 @@ import backtype.storm.spout.SpoutOutputCollector; import backtype.storm.utils.Utils; import com.google.common.collect.ImmutableMap; -import kafka.api.FetchRequestBuilder; import kafka.api.OffsetRequest; import kafka.javaapi.consumer.SimpleConsumer; import kafka.javaapi.message.ByteBufferMessageSet; @@ -17,7 +16,6 @@ import org.slf4j.LoggerFactory; import storm.kafka.KafkaSpout.EmitState; import storm.kafka.KafkaSpout.MessageAndRealOffset; -import storm.kafka.trident.KafkaUtils; import storm.kafka.trident.MaxMetric; import java.util.*; @@ -132,15 +130,8 @@ public EmitState next(SpoutOutputCollector collector) { } private void fill() { - //LOG.info("Fetching from Kafka: " + _consumer.host() + ":" + _partition.partition + " from offset " + _emittedToOffset); long start = System.nanoTime(); - ByteBufferMessageSet msgs = _consumer.fetch( - new FetchRequestBuilder().addFetch( - _spoutConfig.topic, - _partition.partition, - _emittedToOffset, - _spoutConfig.fetchSizeBytes).build()).messageSet(_spoutConfig.topic, - _partition.partition); + ByteBufferMessageSet msgs = KafkaUtils.fetchMessages(_spoutConfig, _consumer, _partition, _emittedToOffset); long end = System.nanoTime(); long millis = (end - start) / 1000000; _fetchAPILatencyMax.update(millis); diff --git a/src/jvm/storm/kafka/trident/Coordinator.java b/src/jvm/storm/kafka/trident/Coordinator.java index d97feedaa7..f67acaafd4 100644 --- a/src/jvm/storm/kafka/trident/Coordinator.java +++ b/src/jvm/storm/kafka/trident/Coordinator.java @@ -1,5 +1,6 @@ package storm.kafka.trident; +import storm.kafka.KafkaUtils; import storm.trident.spout.IOpaquePartitionedTridentSpout; import storm.trident.spout.IPartitionedTridentSpout; diff --git a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java index ab4ec6393f..eceba474cb 100644 --- a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java +++ b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java @@ -7,8 +7,6 @@ import backtype.storm.task.TopologyContext; import backtype.storm.utils.Utils; import com.google.common.collect.ImmutableMap; -import kafka.api.FetchRequest; -import kafka.api.FetchRequestBuilder; import kafka.javaapi.consumer.SimpleConsumer; import kafka.javaapi.message.ByteBufferMessageSet; import kafka.message.Message; @@ -16,13 +14,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import storm.kafka.DynamicPartitionConnections; +import storm.kafka.FailedFetchException; +import storm.kafka.KafkaUtils; import storm.kafka.Partition; import storm.trident.operation.TridentCollector; import storm.trident.spout.IOpaquePartitionedTridentSpout; import storm.trident.spout.IPartitionedTridentSpout; import storm.trident.topology.TransactionAttempt; -import java.net.ConnectException; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -103,16 +102,7 @@ private Map doEmitNewPartitionBatch(SimpleConsumer consumer, Partition partition } offset = KafkaUtils.getOffset(consumer, _config.topic, partition.partition, startTime); } - ByteBufferMessageSet msgs; - try { - msgs = fetchMessages(consumer, partition, offset); - } catch (Exception e) { - if (e instanceof ConnectException) { - throw new FailedFetchException(e); - } else { - throw new RuntimeException(e); - } - } + ByteBufferMessageSet msgs = fetchMessages(consumer, partition, offset); long endoffset = offset; for (MessageAndOffset msg : msgs) { emit(collector, msg.message()); @@ -130,11 +120,8 @@ private Map doEmitNewPartitionBatch(SimpleConsumer consumer, Partition partition } private ByteBufferMessageSet fetchMessages(SimpleConsumer consumer, Partition partition, long offset) { - ByteBufferMessageSet msgs; long start = System.nanoTime(); - FetchRequestBuilder builder = new FetchRequestBuilder(); - FetchRequest fetchRequest = builder.addFetch(_config.topic, partition.partition, offset, _config.fetchSizeBytes).clientId(_config.clientId).build(); - msgs = consumer.fetch(fetchRequest).messageSet(_config.topic, partition.partition); + ByteBufferMessageSet msgs = KafkaUtils.fetchMessages(_config, consumer, partition, offset); long end = System.nanoTime(); long millis = (end - start) / 1000000; _kafkaMeanFetchLatencyMetric.update(millis); diff --git a/src/test/storm/kafka/KafkaTestBroker.java b/src/test/storm/kafka/KafkaTestBroker.java new file mode 100644 index 0000000000..7019c86403 --- /dev/null +++ b/src/test/storm/kafka/KafkaTestBroker.java @@ -0,0 +1,52 @@ +package storm.kafka; + +import com.netflix.curator.framework.CuratorFramework; +import com.netflix.curator.framework.CuratorFrameworkFactory; +import com.netflix.curator.retry.ExponentialBackoffRetry; +import com.netflix.curator.test.TestingServer; +import kafka.server.KafkaServerStartable; + +import java.util.Properties; + +/** + * Date: 11/01/2014 + * Time: 13:15 + */ +public class KafkaTestBroker { + + private final int port = 49123; + private KafkaServerStartable kafka; + private TestingServer server; + private String zookeeperConnectionString; + + public KafkaTestBroker() { + try { + server = new TestingServer(); + zookeeperConnectionString = server.getConnectString(); + ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3); + CuratorFramework zookeeper = CuratorFrameworkFactory.newClient(zookeeperConnectionString, retryPolicy); + zookeeper.start(); + Properties p = new Properties(); + p.setProperty("zookeeper.connect", zookeeperConnectionString); + p.setProperty("broker.id", "0"); + p.setProperty("port", "" + port); + kafka.server.KafkaConfig config = new kafka.server.KafkaConfig(p); + kafka = new KafkaServerStartable(config); + kafka.startup(); + } catch (Exception ex) { + throw new RuntimeException("Could not start test broker", ex); + } + } + + public String getBrokerConnectionString() { + return "localhost:" + port; + } + + public int getPort() { + return port; + } + + public void shutdown() { + kafka.shutdown(); + } +} diff --git a/src/test/storm/kafka/KafkaUtilsTest.java b/src/test/storm/kafka/KafkaUtilsTest.java new file mode 100644 index 0000000000..506789c35a --- /dev/null +++ b/src/test/storm/kafka/KafkaUtilsTest.java @@ -0,0 +1,90 @@ +package storm.kafka; + +import backtype.storm.utils.Utils; +import kafka.api.OffsetRequest; +import kafka.javaapi.consumer.SimpleConsumer; +import kafka.javaapi.message.ByteBufferMessageSet; +import kafka.javaapi.producer.Producer; +import kafka.producer.KeyedMessage; +import kafka.producer.ProducerConfig; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import storm.kafka.trident.GlobalPartitionInformation; + +import java.util.Properties; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class KafkaUtilsTest { + + private KafkaTestBroker broker; + private SimpleConsumer simpleConsumer; + private KafkaConfig config; + + @Before + public void setup() { + broker = new KafkaTestBroker(); + GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(); + globalPartitionInformation.addPartition(0, Broker.fromString(broker.getBrokerConnectionString())); + BrokerHosts brokerHosts = new StaticHosts(globalPartitionInformation); + config = new KafkaConfig(brokerHosts, "testTopic"); + simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient"); + } + + @After + public void shutdown() { + broker.shutdown(); + } + + + @Test(expected = FailedFetchException.class) + public void topicDoesNotExist() throws Exception { + KafkaUtils.fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), 0); + } + + @Test(expected = FailedFetchException.class) + public void brokerIsDown() throws Exception { + int port = broker.getPort(); + broker.shutdown(); + SimpleConsumer simpleConsumer = new SimpleConsumer("localhost", port, 100, 1024, "testClient"); + KafkaUtils.fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), OffsetRequest.LatestTime()); + } + + @Test + public void fetchMessage() throws Exception { + long lastOffset = KafkaUtils.getOffset(simpleConsumer, config.topic, 0, OffsetRequest.EarliestTime()); + sendMessageAndAssertValueForOffset(lastOffset); + } + + @Test(expected = FailedFetchException.class) + public void fetchMessagesWithInvalidOffsetAndDefaultHandlingDisabled() throws Exception { + config.useStartOffsetTimeIfOffsetOutOfRange = false; + sendMessageAndAssertValueForOffset(-99); + } + + @Test + public void fetchMessagesWithInvalidOffsetAndDefaultHandlingEnabled() throws Exception { + sendMessageAndAssertValueForOffset(-99); + } + + private String createTopicAndSendMessage() { + Properties p = new Properties(); + p.setProperty("metadata.broker.list", "localhost:49123"); + p.setProperty("serializer.class", "kafka.serializer.StringEncoder"); + ProducerConfig producerConfig = new ProducerConfig(p); + Producer producer = new Producer(producerConfig); + String value = "value"; + producer.send(new KeyedMessage(config.topic, value)); + return value; + } + + private void sendMessageAndAssertValueForOffset(long offset) { + String value = createTopicAndSendMessage(); + ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), offset); + String message = new String(Utils.toByteArray(messageAndOffsets.iterator().next().message().payload())); + assertThat(message, is(equalTo(value))); + } +} From 80005ba3916e2a05c82649dd67f11b0f813c41b6 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 12 Jan 2014 18:21:56 +0000 Subject: [PATCH 251/349] added OFFSET_METADATA_TOO_LARGE error --- src/jvm/storm/kafka/KafkaError.java | 5 +-- src/test/storm/kafka/KafkaErrorTest.java | 39 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 src/test/storm/kafka/KafkaErrorTest.java diff --git a/src/jvm/storm/kafka/KafkaError.java b/src/jvm/storm/kafka/KafkaError.java index 260ab91568..a67335c3c1 100644 --- a/src/jvm/storm/kafka/KafkaError.java +++ b/src/jvm/storm/kafka/KafkaError.java @@ -17,10 +17,11 @@ public enum KafkaError { REPLICA_NOT_AVAILABLE, MESSAGE_SIZE_TOO_LARGE, STALE_CONTROLLER_EPOCH, + OFFSET_METADATA_TOO_LARGE, UNKNOWN; - public static KafkaError getError(short errorCode) { - if (errorCode < 0) { + public static KafkaError getError(int errorCode) { + if (errorCode < 0 || errorCode >= UNKNOWN.ordinal()) { return UNKNOWN; } else { return values()[errorCode]; diff --git a/src/test/storm/kafka/KafkaErrorTest.java b/src/test/storm/kafka/KafkaErrorTest.java new file mode 100644 index 0000000000..e5f9db280f --- /dev/null +++ b/src/test/storm/kafka/KafkaErrorTest.java @@ -0,0 +1,39 @@ +package storm.kafka; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * Date: 12/01/2014 + * Time: 18:09 + */ +public class KafkaErrorTest { + + @Test + public void getError() { + assertThat(KafkaError.getError(0), is(equalTo(KafkaError.NO_ERROR))); + } + + @Test + public void offsetMetaDataTooLarge() { + assertThat(KafkaError.getError(12), is(equalTo(KafkaError.OFFSET_METADATA_TOO_LARGE))); + } + + @Test + public void unknownNegative() { + assertThat(KafkaError.getError(-1), is(equalTo(KafkaError.UNKNOWN))); + } + + @Test + public void unknownPositive() { + assertThat(KafkaError.getError(75), is(equalTo(KafkaError.UNKNOWN))); + } + + @Test + public void unknown() { + assertThat(KafkaError.getError(13), is(equalTo(KafkaError.UNKNOWN))); + } +} From 47387185b1c8b61b24c8c3be1675e4e4f2c670c6 Mon Sep 17 00:00:00 2001 From: Derek Dagit Date: Sun, 12 Jan 2014 00:48:52 +0000 Subject: [PATCH 252/349] Do not rely on spaces to separate command args --- .../clj/backtype/storm/daemon/supervisor.clj | 38 +++++++++++-------- storm-core/src/clj/backtype/storm/util.clj | 4 +- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj index 43cb6fe8b8..9bff2ada0c 100644 --- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj +++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj @@ -447,22 +447,30 @@ stormjar (supervisor-stormjar-path stormroot) storm-conf (read-supervisor-storm-conf conf storm-id) classpath (add-to-classpath (current-classpath) [stormjar]) - childopts (.replaceAll (str (conf WORKER-CHILDOPTS) " " (storm-conf TOPOLOGY-WORKER-CHILDOPTS)) - "%ID%" - (str port)) + worker-childopts (when-let [s (conf WORKER-CHILDOPTS)] + (.replaceAll s "%ID%" (str port))) + topo-worker-childopts (when-let [s (conf TOPOLOGY-WORKER-CHILDOPTS)] + (.replaceAll s "%ID%" (str port))) logfilename (str "worker-" port ".log") - command (str "java -server " childopts - " -Djava.library.path=" (conf JAVA-LIBRARY-PATH) - " -Dlogfile.name=" logfilename - " -Dstorm.home=" storm-home - " -Dlogback.configurationFile=" storm-home "/logback/cluster.xml" - " -Dstorm.id=" storm-id - " -Dworker.id=" worker-id - " -Dworker.port=" port - " -cp " classpath " backtype.storm.daemon.worker " - (java.net.URLEncoder/encode storm-id) " " (:assignment-id supervisor) - " " port " " worker-id)] - (log-message "Launching worker with command: " command) + command ["java" + "-server" + worker-childopts + topo-worker-childopts + (str "-Djava.library.path=" (conf JAVA-LIBRARY-PATH)) + (str "-Dlogfile.name=" logfilename) + (str "-Dstorm.home=" storm-home) + (str "-Dlogback.configurationFile=" storm-home "/logback/cluster.xml") + (str "-Dstorm.id=" storm-id) + (str "-Dworker.id=" worker-id) + (str "-Dworker.port=" port) + "-cp" classpath + "backtype.storm.daemon.worker" + (java.net.URLEncoder/encode storm-id) + (:assignment-id supervisor) + port + worker-id] + command (->> command (map str) (filter (complement empty?)))] + (log-message "Launching worker with command: " (clojure.string/join " " command)) (launch-process command :environment {"LD_LIBRARY_PATH" (conf JAVA-LIBRARY-PATH)}) )) diff --git a/storm-core/src/clj/backtype/storm/util.clj b/storm-core/src/clj/backtype/storm/util.clj index 2ab30c7e24..a3d5ebb078 100644 --- a/storm-core/src/clj/backtype/storm/util.clj +++ b/storm-core/src/clj/backtype/storm/util.clj @@ -395,9 +395,7 @@ )) (defnk launch-process [command :environment {}] - (let [command (->> (seq (.split command " ")) - (filter (complement empty?))) - builder (ProcessBuilder. command) + (let [builder (ProcessBuilder. command) process-env (.environment builder)] (doseq [[k v] environment] (.put process-env k v)) From 193301f2d4d8188b5cee04f4cf982636ea6c6ba6 Mon Sep 17 00:00:00 2001 From: Derek Dagit Date: Sun, 12 Jan 2014 00:50:36 +0000 Subject: [PATCH 253/349] URL-decode special chars in topo ids --- storm-core/src/clj/backtype/storm/ui/core.clj | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj index 63a3053d52..e46871ad84 100644 --- a/storm-core/src/clj/backtype/storm/ui/core.clj +++ b/storm-core/src/clj/backtype/storm/ui/core.clj @@ -29,6 +29,7 @@ Nimbus$Client StormTopology GlobalStreamId RebalanceOptions KillOptions]) (:import [java.io File]) + (:import [java.net URLDecoder]) (:require [compojure.route :as route] [compojure.handler :as handler] [ring.util.response :as resp] @@ -1018,32 +1019,33 @@ (GET "/topology/:id" [:as {cookies :cookies} id & m] (let [include-sys? (get-include-sys? cookies)] (try - (-> (topology-page id (:window m) include-sys?) + (-> (topology-page (URLDecoder/decode id) (:window m) include-sys?) (concat [(mk-system-toggle-button include-sys?)]) ui-template) (catch Exception e (resp/redirect "/"))))) (GET "/topology/:id/component/:component" [:as {cookies :cookies} id component & m] (let [include-sys? (get-include-sys? cookies)] - (-> (component-page id component (:window m) include-sys?) + (-> (component-page (URLDecoder/decode id) + component (:window m) include-sys?) (concat [(mk-system-toggle-button include-sys?)]) ui-template))) (POST "/topology/:id/activate" [id] (with-nimbus nimbus - (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id) + (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus (URLDecoder/decode id)) name (.get_name tplg)] (.activate nimbus name) (log-message "Activating topology '" name "'"))) (resp/redirect (str "/topology/" id))) (POST "/topology/:id/deactivate" [id] (with-nimbus nimbus - (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id) + (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus (URLDecoder/decode id)) name (.get_name tplg)] (.deactivate nimbus name) (log-message "Deactivating topology '" name "'"))) (resp/redirect (str "/topology/" id))) (POST "/topology/:id/rebalance/:wait-time" [id wait-time] (with-nimbus nimbus - (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id) + (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus (URLDecoder/decode id)) name (.get_name tplg) options (RebalanceOptions.)] (.set_wait_secs options (Integer/parseInt wait-time)) @@ -1052,7 +1054,7 @@ (resp/redirect (str "/topology/" id))) (POST "/topology/:id/kill/:wait-time" [id wait-time] (with-nimbus nimbus - (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id) + (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus (URLDecoder/decode id)) name (.get_name tplg) options (KillOptions.)] (.set_wait_secs options (Integer/parseInt wait-time)) From c7c7087c4be7fff719e846910af1b18c48d01a38 Mon Sep 17 00:00:00 2001 From: Derek Dagit Date: Sun, 12 Jan 2014 20:34:51 +0000 Subject: [PATCH 254/349] supervisor logs worker command with '-quoted tokens --- storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj index 9bff2ada0c..ea2b4607da 100644 --- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj +++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj @@ -469,8 +469,9 @@ (:assignment-id supervisor) port worker-id] - command (->> command (map str) (filter (complement empty?)))] - (log-message "Launching worker with command: " (clojure.string/join " " command)) + command (->> command (map str) (filter (complement empty?))) + shell-cmd (->> command (map #(str \' % \')) (clojure.string/join " "))] + (log-message "Launching worker with command: " shell-cmd) (launch-process command :environment {"LD_LIBRARY_PATH" (conf JAVA-LIBRARY-PATH)}) )) From 6d1ad6a6a3835294a564ab63b1d6399ee7843e18 Mon Sep 17 00:00:00 2001 From: Derek Dagit Date: Sun, 12 Jan 2014 20:46:46 +0000 Subject: [PATCH 255/349] Do not encode,decode topo id when launching worker --- storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 2 +- storm-core/src/clj/backtype/storm/daemon/worker.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj index ea2b4607da..a38ee5c638 100644 --- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj +++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj @@ -465,7 +465,7 @@ (str "-Dworker.port=" port) "-cp" classpath "backtype.storm.daemon.worker" - (java.net.URLEncoder/encode storm-id) + storm-id (:assignment-id supervisor) port worker-id] diff --git a/storm-core/src/clj/backtype/storm/daemon/worker.clj b/storm-core/src/clj/backtype/storm/daemon/worker.clj index 3055b66bc6..0d1f6c6921 100644 --- a/storm-core/src/clj/backtype/storm/daemon/worker.clj +++ b/storm-core/src/clj/backtype/storm/daemon/worker.clj @@ -451,4 +451,4 @@ (defn -main [storm-id assignment-id port-str worker-id] (let [conf (read-storm-config)] (validate-distributed-mode! conf) - (mk-worker conf nil (java.net.URLDecoder/decode storm-id) assignment-id (Integer/parseInt port-str) worker-id))) + (mk-worker conf nil storm-id assignment-id (Integer/parseInt port-str) worker-id))) From c970225dc43721e1a7827408df7542aed94e11cb Mon Sep 17 00:00:00 2001 From: Derek Dagit Date: Wed, 15 Jan 2014 02:02:41 +0000 Subject: [PATCH 256/349] Support *.worker.childopts as list or as string --- .../clj/backtype/storm/daemon/supervisor.clj | 45 ++++++++++--------- storm-core/src/jvm/backtype/storm/Config.java | 5 +-- .../jvm/backtype/storm/ConfigValidation.java | 17 +++++++ .../test/clj/backtype/storm/config_test.clj | 20 +++++++++ 4 files changed, 64 insertions(+), 23 deletions(-) diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj index a38ee5c638..1630766eb6 100644 --- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj +++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj @@ -438,6 +438,11 @@ (FileUtils/moveDirectory (File. tmproot) (File. stormroot)) )) +(defn- substitute-worker-childopts [value port] + (let [sub-fn (fn [s] (.replaceAll s "%ID%" (str port)))] + (if (list? value) + (map sub-fn value) + (-> value sub-fn (.split " "))))) (defmethod launch-worker :distributed [supervisor storm-id port worker-id] @@ -448,27 +453,27 @@ storm-conf (read-supervisor-storm-conf conf storm-id) classpath (add-to-classpath (current-classpath) [stormjar]) worker-childopts (when-let [s (conf WORKER-CHILDOPTS)] - (.replaceAll s "%ID%" (str port))) - topo-worker-childopts (when-let [s (conf TOPOLOGY-WORKER-CHILDOPTS)] - (.replaceAll s "%ID%" (str port))) + (substitute-worker-childopts s port)) + topo-worker-childopts (when-let [s (storm-conf TOPOLOGY-WORKER-CHILDOPTS)] + (substitute-worker-childopts s port)) logfilename (str "worker-" port ".log") - command ["java" - "-server" - worker-childopts - topo-worker-childopts - (str "-Djava.library.path=" (conf JAVA-LIBRARY-PATH)) - (str "-Dlogfile.name=" logfilename) - (str "-Dstorm.home=" storm-home) - (str "-Dlogback.configurationFile=" storm-home "/logback/cluster.xml") - (str "-Dstorm.id=" storm-id) - (str "-Dworker.id=" worker-id) - (str "-Dworker.port=" port) - "-cp" classpath - "backtype.storm.daemon.worker" - storm-id - (:assignment-id supervisor) - port - worker-id] + command (concat + ["java" "-server"] + worker-childopts + topo-worker-childopts + [(str "-Djava.library.path=" (conf JAVA-LIBRARY-PATH)) + (str "-Dlogfile.name=" logfilename) + (str "-Dstorm.home=" storm-home) + (str "-Dlogback.configurationFile=" storm-home "/logback/cluster.xml") + (str "-Dstorm.id=" storm-id) + (str "-Dworker.id=" worker-id) + (str "-Dworker.port=" port) + "-cp" classpath + "backtype.storm.daemon.worker" + storm-id + (:assignment-id supervisor) + port + worker-id]) command (->> command (map str) (filter (complement empty?))) shell-cmd (->> command (map #(str \' % \')) (clojure.string/join " "))] (log-message "Launching worker with command: " shell-cmd) diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java index 281ae525df..1fc0d78788 100644 --- a/storm-core/src/jvm/backtype/storm/Config.java +++ b/storm-core/src/jvm/backtype/storm/Config.java @@ -459,8 +459,7 @@ public class Config extends HashMap { * with an identifier for this worker. */ public static final String WORKER_CHILDOPTS = "worker.childopts"; - public static final Object WORKER_CHILDOPTS_SCHEMA = String.class; - + public static final Object WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator; /** * How often this worker should heartbeat to the supervisor. @@ -662,7 +661,7 @@ public class Config extends HashMap { * Topology-specific options for the worker child process. This is used in addition to WORKER_CHILDOPTS. */ public static final String TOPOLOGY_WORKER_CHILDOPTS="topology.worker.childopts"; - public static final Object TOPOLOGY_WORKER_CHILDOPTS_SCHEMA = String.class; + public static final Object TOPOLOGY_WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator; /** * This config is available for TransactionalSpouts, and contains the id ( a String) for diff --git a/storm-core/src/jvm/backtype/storm/ConfigValidation.java b/storm-core/src/jvm/backtype/storm/ConfigValidation.java index 15ef6ba129..3accb820ef 100644 --- a/storm-core/src/jvm/backtype/storm/ConfigValidation.java +++ b/storm-core/src/jvm/backtype/storm/ConfigValidation.java @@ -135,4 +135,21 @@ public void validateField(String name, Object o) throws IllegalArgumentException "Field " + name + " must be an Iterable containing only Strings or Maps of Strings"); } }; + + /** + * Validates a String or a list of Strings + */ + public static Object StringOrStringListValidator = new FieldValidator() { + + private FieldValidator fv = FieldListValidatorFactory(String.class); + + @Override + public void validateField(String name, Object o) throws IllegalArgumentException { + if (o == null || o instanceof String) { + // A null value or a String value is acceptable + return; + } + this.fv.validateField(name, o); + } + }; } diff --git a/storm-core/test/clj/backtype/storm/config_test.clj b/storm-core/test/clj/backtype/storm/config_test.clj index 93c7df9397..01f788ba73 100644 --- a/storm-core/test/clj/backtype/storm/config_test.clj +++ b/storm-core/test/clj/backtype/storm/config_test.clj @@ -83,3 +83,23 @@ (catch Exception e e)))) (is (thrown-cause? java.lang.IllegalArgumentException (.validateField validator "test" 42))))) + +(deftest test-worker-childopts-is-string-or-string-list + (let [pass-cases [nil "some string" ["some" "string" "list"]]] + (testing "worker.childopts validates" + (let [validator (CONFIG-SCHEMA-MAP WORKER-CHILDOPTS)] + (doseq [value pass-cases] + (is (nil? (try + (.validateField validator "test" value) + (catch Exception e e))))) + (is (thrown-cause? java.lang.IllegalArgumentException + (.validateField validator "test" 42))))) + + (testing "topology.worker.childopts validates" + (let [validator (CONFIG-SCHEMA-MAP TOPOLOGY-WORKER-CHILDOPTS)] + (doseq [value pass-cases] + (is (nil? (try + (.validateField validator "test" value) + (catch Exception e e))))) + (is (thrown-cause? java.lang.IllegalArgumentException + (.validateField validator "test" 42))))))) From 2fb66fa3be27f3495fbbe78831565a1edce12af1 Mon Sep 17 00:00:00 2001 From: Derek Dagit Date: Wed, 15 Jan 2014 03:10:35 +0000 Subject: [PATCH 257/349] Add test for handling *.worker.childopts in supvor --- pom.xml | 7 ++ storm-core/pom.xml | 5 ++ .../clj/backtype/storm/supervisor_test.clj | 64 ++++++++++++++++++- 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3bd3362b8e..3a0c26bd4c 100644 --- a/pom.xml +++ b/pom.xml @@ -166,6 +166,7 @@ 0.2.3 1.9.5 0.3.0 + 2.1.3 @@ -410,6 +411,12 @@ ${mockito.version} test + + org.clojars.runa + conjure + ${conjure.version} + test + org.apache.storm libthrift7 diff --git a/storm-core/pom.xml b/storm-core/pom.xml index d333b88c37..f1a74c6f28 100644 --- a/storm-core/pom.xml +++ b/storm-core/pom.xml @@ -145,6 +145,11 @@ mockito-all test + + org.clojars.runa + conjure + test + src/jvm diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj index 9f9def98eb..54a3ee9cd5 100644 --- a/storm-core/test/clj/backtype/storm/supervisor_test.clj +++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj @@ -15,10 +15,12 @@ ;; limitations under the License. (ns backtype.storm.supervisor-test (:use [clojure test]) + (:require [clojure [string :as string]]) (:import [backtype.storm.testing TestWordCounter TestWordSpout TestGlobalCount TestAggregatesCounter]) - (:use [backtype.storm bootstrap testing]) + (:use [backtype.storm bootstrap config testing]) (:use [backtype.storm.daemon common]) (:require [backtype.storm.daemon [worker :as worker] [supervisor :as supervisor]]) + (:use [conjure core]) ) (bootstrap) @@ -239,6 +241,66 @@ ))) +(deftest test-worker-launch-command + (testing "*.worker.childopts configuration" + (let [mock-port "42" + mock-storm-id "fake-storm-id" + mock-worker-id "fake-worker-id" + mock-cp "mock-classpath" + exp-args-fn (fn [opts topo-opts] + (concat ["java" "-server"] + opts + topo-opts + ["-Djava.library.path=" + (str "-Dlogfile.name=worker-" mock-port ".log") + "-Dstorm.home=" + "-Dlogback.configurationFile=/logback/cluster.xml" + (str "-Dstorm.id=" mock-storm-id) + (str "-Dworker.id=" mock-worker-id) + (str "-Dworker.port=" mock-port) + "-cp" mock-cp + "backtype.storm.daemon.worker" + mock-storm-id + mock-port + mock-worker-id]))] + (testing "testing *.worker.childopts as strings with extra spaces" + (let [string-opts "-Dfoo=bar -Xmx1024m" + topo-string-opts "-Dkau=aux -Xmx2048m" + exp-args (exp-args-fn ["-Dfoo=bar" "-Xmx1024m"] + ["-Dkau=aux" "-Xmx2048m"]) + mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed + WORKER-CHILDOPTS string-opts}}] + (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS + topo-string-opts} + add-to-classpath mock-cp + supervisor-stormdist-root nil + launch-process nil] + (supervisor/launch-worker mock-supervisor + mock-storm-id + mock-port + mock-worker-id) + (verify-first-call-args-for-indices launch-process + [0] + exp-args)))) + (testing "testing *.worker.childopts as list of strings, with spaces in values" + (let [list-opts '("-Dopt1='this has a space in it'" "-Xmx1024m") + topo-list-opts '("-Dopt2='val with spaces'" "-Xmx2048m") + exp-args (exp-args-fn list-opts topo-list-opts) + mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed + WORKER-CHILDOPTS list-opts}}] + (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS + topo-list-opts} + add-to-classpath mock-cp + supervisor-stormdist-root nil + launch-process nil] + (supervisor/launch-worker mock-supervisor + mock-storm-id + mock-port + mock-worker-id) + (verify-first-call-args-for-indices launch-process + [0] + exp-args))))))) + (deftest test-workers-go-bananas ;; test that multiple workers are started for a port, and test that ;; supervisor shuts down propertly (doesn't shutdown the most From 5b764cd9138dd93e7382e3472b9d3d33d4b286a3 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sat, 18 Jan 2014 15:25:36 +0000 Subject: [PATCH 258/349] calculate start offset for new topology consistently --- src/jvm/storm/kafka/KafkaUtils.java | 9 +++++++++ src/jvm/storm/kafka/PartitionManager.java | 10 +++++----- src/jvm/storm/kafka/trident/TridentKafkaEmitter.java | 6 +----- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/jvm/storm/kafka/KafkaUtils.java b/src/jvm/storm/kafka/KafkaUtils.java index 5094f14ee7..d86519c524 100644 --- a/src/jvm/storm/kafka/KafkaUtils.java +++ b/src/jvm/storm/kafka/KafkaUtils.java @@ -36,6 +36,15 @@ public static IBrokerReader makeBrokerReader(Map stormConf, KafkaConfig conf) { } } + + public static long getOffset(SimpleConsumer consumer, String topic, int partition, KafkaConfig config) { + long startOffsetTime = kafka.api.OffsetRequest.LatestTime(); + if ( config.forceFromStart ) { + startOffsetTime = config.startOffsetTime; + } + return getOffset(consumer, topic, partition, startOffsetTime); + } + public static long getOffset(SimpleConsumer consumer, String topic, int partition, long startOffsetTime) { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map requestInfo = new HashMap(); diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 0861c255cd..f12c0d991e 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -73,12 +73,12 @@ public PartitionManager(DynamicPartitionConnections connections, String topology LOG.warn("Error reading and/or parsing at ZkNode: " + path, e); } - if (!topologyInstanceId.equals(jsonTopologyId) && spoutConfig.forceFromStart) { + if (jsonTopologyId == null || jsonOffset == null) { // failed to parse JSON? + _committedTo = KafkaUtils.getOffset(_consumer, spoutConfig.topic, id.partition, spoutConfig); + LOG.info("No partition information found, using configuration to determine offset"); + } else if (!topologyInstanceId.equals(jsonTopologyId) && spoutConfig.forceFromStart) { _committedTo = KafkaUtils.getOffset(_consumer, spoutConfig.topic, id.partition, spoutConfig.startOffsetTime); - LOG.info("Using startOffsetTime to choose last commit offset."); - } else if (jsonTopologyId == null || jsonOffset == null) { // failed to parse JSON? - _committedTo = KafkaUtils.getOffset(_consumer, spoutConfig.topic, id.partition, kafka.api.OffsetRequest.LatestTime()); - LOG.info("Setting last commit offset to HEAD."); + LOG.info("Topology change detected and reset from start forced, using configuration to determine offset"); } else { _committedTo = jsonOffset; LOG.info("Read last commit offset from zookeeper: " + _committedTo + "; old topology_id: " + jsonTopologyId + " - new topology_id: " + topologyInstanceId ); diff --git a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java index eceba474cb..fbbbd4bdce 100644 --- a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java +++ b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java @@ -96,11 +96,7 @@ private Map doEmitNewPartitionBatch(SimpleConsumer consumer, Partition partition offset = (Long) lastMeta.get("nextOffset"); } } else { - long startTime = kafka.api.OffsetRequest.LatestTime(); - if (_config.forceFromStart) { - startTime = _config.startOffsetTime; - } - offset = KafkaUtils.getOffset(consumer, _config.topic, partition.partition, startTime); + offset = KafkaUtils.getOffset(consumer, _config.topic, partition.partition, _config); } ByteBufferMessageSet msgs = fetchMessages(consumer, partition, offset); long endoffset = offset; From f789091534be95c103890be6539bbfc5faf69b37 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sat, 18 Jan 2014 15:51:44 +0000 Subject: [PATCH 259/349] calculate start offset for new topology consistently --- src/test/storm/kafka/KafkaUtilsTest.java | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/test/storm/kafka/KafkaUtilsTest.java b/src/test/storm/kafka/KafkaUtilsTest.java index 506789c35a..20a4221644 100644 --- a/src/test/storm/kafka/KafkaUtilsTest.java +++ b/src/test/storm/kafka/KafkaUtilsTest.java @@ -70,6 +70,26 @@ public void fetchMessagesWithInvalidOffsetAndDefaultHandlingEnabled() throws Exc sendMessageAndAssertValueForOffset(-99); } + @Test + public void getOffsetFromConfigAndDontForceFromStart() { + config.forceFromStart = false; + config.startOffsetTime = OffsetRequest.EarliestTime(); + createTopicAndSendMessage(); + long latestOffset = KafkaUtils.getOffset(simpleConsumer, config.topic, 0, OffsetRequest.LatestTime()); + long offsetFromConfig = KafkaUtils.getOffset(simpleConsumer, config.topic, 0, config); + assertThat(latestOffset, is(equalTo(offsetFromConfig))); + } + + @Test + public void getOffsetFromConfigAndFroceFromStart() { + config.forceFromStart = true; + config.startOffsetTime = OffsetRequest.EarliestTime(); + createTopicAndSendMessage(); + long earliestOffset = KafkaUtils.getOffset(simpleConsumer, config.topic, 0, OffsetRequest.EarliestTime()); + long offsetFromConfig = KafkaUtils.getOffset(simpleConsumer, config.topic, 0, config); + assertThat(earliestOffset, is(equalTo(offsetFromConfig))); + } + private String createTopicAndSendMessage() { Properties p = new Properties(); p.setProperty("metadata.broker.list", "localhost:49123"); From 0a6203acc6a2765049af6913b8deb070fb63a244 Mon Sep 17 00:00:00 2001 From: Edison Xu Date: Wed, 22 Jan 2014 15:07:04 +0800 Subject: [PATCH 260/349] fix trident keep refreshing partition issue --- .../trident/GlobalPartitionInformation.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java b/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java index a7900092f2..e73d6b066b 100644 --- a/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java +++ b/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java @@ -64,4 +64,30 @@ public void remove() { } }; } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((partitionMap == null) ? 0 : partitionMap.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + GlobalPartitionInformation other = (GlobalPartitionInformation) obj; + if (partitionMap == null) { + if (other.partitionMap != null) + return false; + } else if (!partitionMap.equals(other.partitionMap)) + return false; + return true; + } } From aabcd5846010ff0bc5832520bf8f59e9879728cc Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 26 Jan 2014 21:32:13 +0000 Subject: [PATCH 261/349] use guava equals/hashCode --- src/jvm/storm/kafka/Broker.java | 17 ++++++++---- .../trident/GlobalPartitionInformation.java | 26 +++++++------------ 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/jvm/storm/kafka/Broker.java b/src/jvm/storm/kafka/Broker.java index 66e6112a89..2451eee454 100644 --- a/src/jvm/storm/kafka/Broker.java +++ b/src/jvm/storm/kafka/Broker.java @@ -1,6 +1,7 @@ package storm.kafka; import java.io.Serializable; +import com.google.common.base.Objects; public class Broker implements Serializable, Comparable { public final String host; @@ -16,14 +17,20 @@ public Broker(String host) { } @Override - public boolean equals(Object o) { - Broker other = (Broker) o; - return host.equals(other.host) && port == other.port; + public int hashCode() { + return Objects.hashCode(host, port); } @Override - public int hashCode() { - return host.hashCode(); + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final Broker other = (Broker) obj; + return Objects.equal(this.host, other.host) && Objects.equal(this.port, other.port); } @Override diff --git a/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java b/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java index e73d6b066b..6f82f62ddb 100644 --- a/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java +++ b/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java @@ -1,12 +1,13 @@ package storm.kafka.trident; -import storm.kafka.Broker; import storm.kafka.Broker; import storm.kafka.Partition; import java.io.Serializable; import java.util.*; +import com.google.common.base.Objects; + /** * Date: 14/05/2013 * Time: 19:18 @@ -67,27 +68,18 @@ public void remove() { @Override public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result - + ((partitionMap == null) ? 0 : partitionMap.hashCode()); - return result; + return Objects.hashCode(partitionMap); } @Override public boolean equals(Object obj) { - if (this == obj) + if (this == obj) { return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - GlobalPartitionInformation other = (GlobalPartitionInformation) obj; - if (partitionMap == null) { - if (other.partitionMap != null) - return false; - } else if (!partitionMap.equals(other.partitionMap)) + } + if (obj == null || getClass() != obj.getClass()) { return false; - return true; + } + final GlobalPartitionInformation other = (GlobalPartitionInformation) obj; + return Objects.equal(this.partitionMap, other.partitionMap); } } From e9357eca4881704c398acbe60dda228ad4c7beec Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 2 Feb 2014 19:24:47 +0000 Subject: [PATCH 262/349] added profile for scala 2.10 build --- pom.xml | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 8dfd316b36..8defde4da2 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,21 @@ scm:git:ssh://git@github.com/wurstmeister/storm-kafka-0.8-plus.git https://github.com/wurstmeister/storm-kafka-0.8-plus + + 2.9.2 + kafka_2.9.2 + + + + + maven-jar-plugin + 2.4 + + ${envClassifier} + + + src/jvm src/test @@ -87,7 +101,7 @@ org.scala-lang scala-library - 2.9.2 + ${scalaVersion} junit @@ -140,7 +154,7 @@ org.apache.kafka - kafka_2.9.2 + ${kafkaArtifact} 0.8.0 @@ -179,4 +193,15 @@ https://clojars.org/repo + + + + Scala-2.10 + + 2.10.3 + kafka_2.10 + scala_2.10 + + + From 9368ac447aeb473f926302137dbcdf914d06cd32 Mon Sep 17 00:00:00 2001 From: yerenkow Date: Mon, 3 Feb 2014 11:05:52 +0200 Subject: [PATCH 263/349] Fix for launching jar when some JVM opts are specified, now corrected and using shlex library --- bin/storm | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/storm b/bin/storm index aee46cbfa3..6100f88abf 100755 --- a/bin/storm +++ b/bin/storm @@ -22,6 +22,7 @@ import random import subprocess as sub import getopt import re +import shlex def identity(x): return x @@ -45,7 +46,9 @@ if (not os.path.isfile(USER_CONF_DIR + "/storm.yaml")): USER_CONF_DIR = CLUSTER_CONF_DIR CONFIG_OPTS = [] CONFFILE = "" -JAR_JVM_OPTS = os.getenv('STORM_JAR_JVM_OPTS', '') + +#note that JAR_JVM_OPTS is now array instead of string +JAR_JVM_OPTS = shlex.split(os.getenv('STORM_JAR_JVM_OPTS', '')) def get_config_opts(): @@ -155,7 +158,7 @@ def jar(jarfile, klass, *args): jvmtype="-client", extrajars=[jarfile, USER_CONF_DIR, STORM_DIR + "/bin"], args=args, - jvmopts=[' '.join(filter(None, [JAR_JVM_OPTS, "-Dstorm.jar=" + jarfile]))]) + jvmopts=JAR_JVM_OPTS + ["-Dstorm.jar=" + jarfile]) def kill(*args): """Syntax: [storm kill topology-name [-w wait-time-secs]] From a63c0d976aabbb4ff9692ab25aa70aa216a0b0d7 Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Fri, 7 Feb 2014 22:43:32 +0000 Subject: [PATCH 264/349] STORM-70 Upgrade to ZK-3.4.5 and curator-1.3.3 --- pom.xml | 35 +++++++++++++++++-- .../src/clj/backtype/storm/zookeeper.clj | 4 +-- .../src/jvm/backtype/storm/utils/Utils.java | 26 ++++++-------- 3 files changed, 46 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index 7a3a840e40..c66ee22ca4 100644 --- a/pom.xml +++ b/pom.xml @@ -155,7 +155,7 @@ 2.5 1.1 0.4.1 - 1.0.1 + 1.3.3 1.1 0.3.11 0.2.3 @@ -174,7 +174,7 @@ 0.2.3 1.9.5 0.3.0 - + 3.4.5 @@ -284,6 +284,37 @@ clj-time ${clj-time.version} + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + + + com.sun.jmx + jmxri + + + com.sun.jdmk + jmxtools + + + javax.jms + jms + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + com.netflix.curator curator-framework diff --git a/storm-core/src/clj/backtype/storm/zookeeper.clj b/storm-core/src/clj/backtype/storm/zookeeper.clj index 617e69b323..5e70cb1de1 100644 --- a/storm-core/src/clj/backtype/storm/zookeeper.clj +++ b/storm-core/src/clj/backtype/storm/zookeeper.clj @@ -21,7 +21,7 @@ ZooDefs ZooDefs$Ids CreateMode WatchedEvent Watcher$Event Watcher$Event$KeeperState Watcher$Event$EventType KeeperException$NodeExistsException]) (:import [org.apache.zookeeper.data Stat]) - (:import [org.apache.zookeeper.server ZooKeeperServer NIOServerCnxn$Factory]) + (:import [org.apache.zookeeper.server ZooKeeperServer NIOServerCnxnFactory]) (:import [java.net InetSocketAddress BindException]) (:import [java.io File]) (:import [backtype.storm.utils Utils ZookeeperAuthInfo]) @@ -156,7 +156,7 @@ (let [localfile (File. localdir) zk (ZooKeeperServer. localfile localfile 2000) [retport factory] (loop [retport (if port port 2000)] - (if-let [factory-tmp (try-cause (NIOServerCnxn$Factory. (InetSocketAddress. retport)) + (if-let [factory-tmp (try-cause (doto (NIOServerCnxnFactory.) (.configure (InetSocketAddress. retport) 0)) (catch BindException e (when (> (inc retport) (if port port 65535)) (throw (RuntimeException. "No port is available to launch an inprocess zookeeper.")))))] diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java index 8a05fe4952..0e047281a3 100644 --- a/storm-core/src/jvm/backtype/storm/utils/Utils.java +++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java @@ -335,22 +335,18 @@ public static CuratorFramework newCurator(Map conf, List servers, Object serverPorts.add(zkServer + ":" + Utils.getInt(port)); } String zkStr = StringUtils.join(serverPorts, ",") + root; - try { - CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder() - .connectString(zkStr) - .connectionTimeoutMs(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT))) - .sessionTimeoutMs(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT))) - .retryPolicy(new BoundedExponentialBackoffRetry( - Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)), - Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), - Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING)))); - if(auth!=null && auth.scheme!=null) { - builder = builder.authorization(auth.scheme, auth.payload); - } - return builder.build(); - } catch (IOException e) { - throw new RuntimeException(e); + CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder() + .connectString(zkStr) + .connectionTimeoutMs(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT))) + .sessionTimeoutMs(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT))) + .retryPolicy(new BoundedExponentialBackoffRetry( + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)), + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING)))); + if(auth!=null && auth.scheme!=null) { + builder = builder.authorization(auth.scheme, auth.payload); } + return builder.build(); } public static CuratorFramework newCurator(Map conf, List servers, Object port) { From 37aa116d7aba38c7600b669589bc9458e328bb4a Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sat, 8 Feb 2014 08:32:25 +0000 Subject: [PATCH 265/349] preparing for tag --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8defde4da2..6de3e7f664 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ net.wurstmeister.storm storm-kafka-0.8-plus jar - 0.3.0-SNAPSHOT + 0.3.0 storm-kafka-0.8-plus Storm module for kafka > 0.8 From 312408a5c2d22bc2a1578d940c92417e05f94412 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sat, 8 Feb 2014 08:47:12 +0000 Subject: [PATCH 266/349] update to next snapshot version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6de3e7f664..910041ad90 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ net.wurstmeister.storm storm-kafka-0.8-plus jar - 0.3.0 + 0.4.0-SNAPSHOT storm-kafka-0.8-plus Storm module for kafka > 0.8 From ffc7a81bfba60c735dd6801af4f5e8db3812658c Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Sun, 9 Feb 2014 22:35:48 -0500 Subject: [PATCH 267/349] [maven-release-plugin] prepare release v0.9.1-incubating --- pom.xml | 4 ++-- storm-buildtools/maven-shade-clojure-transformer/pom.xml | 2 +- storm-core/pom.xml | 4 ++-- storm-dist/binary/pom.xml | 2 +- storm-dist/source/pom.xml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 720fab3b9c..a274e3f103 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ org.apache.storm storm - 0.9.1-incubating-SNAPSHOT + 0.9.1-incubating pom Storm Distributed and fault-tolerant realtime computation @@ -135,7 +135,7 @@ scm:git:https://git-wip-us.apache.org/repos/asf/incubator-storm.git scm:git:https://git-wip-us.apache.org/repos/asf/incubator-storm.git - HEAD + v0.9.1-incubating https://git-wip-us.apache.org/repos/asf/incubator-storm diff --git a/storm-buildtools/maven-shade-clojure-transformer/pom.xml b/storm-buildtools/maven-shade-clojure-transformer/pom.xml index 3b8cd9796a..bfaafc097f 100644 --- a/storm-buildtools/maven-shade-clojure-transformer/pom.xml +++ b/storm-buildtools/maven-shade-clojure-transformer/pom.xml @@ -21,7 +21,7 @@ storm org.apache.storm - 0.9.1-incubating-SNAPSHOT + 0.9.1-incubating ../../pom.xml diff --git a/storm-core/pom.xml b/storm-core/pom.xml index 20366861ec..af3a338d97 100644 --- a/storm-core/pom.xml +++ b/storm-core/pom.xml @@ -20,7 +20,7 @@ storm org.apache.storm - 0.9.1-incubating-SNAPSHOT + 0.9.1-incubating org.apache.storm storm-core @@ -250,7 +250,7 @@ - + diff --git a/storm-dist/binary/pom.xml b/storm-dist/binary/pom.xml index 54ec172c00..15b76b2aa0 100644 --- a/storm-dist/binary/pom.xml +++ b/storm-dist/binary/pom.xml @@ -21,7 +21,7 @@ storm org.apache.storm - 0.9.1-incubating-SNAPSHOT + 0.9.1-incubating ../../pom.xml org.apache.storm diff --git a/storm-dist/source/pom.xml b/storm-dist/source/pom.xml index d2a4366288..aa6c13859f 100644 --- a/storm-dist/source/pom.xml +++ b/storm-dist/source/pom.xml @@ -21,7 +21,7 @@ storm org.apache.storm - 0.9.1-incubating-SNAPSHOT + 0.9.1-incubating ../../pom.xml org.apache.storm From 9fe14b405da30638be348fd3fdbc639e6a7376e9 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Sun, 9 Feb 2014 22:45:31 -0500 Subject: [PATCH 268/349] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- storm-buildtools/maven-shade-clojure-transformer/pom.xml | 2 +- storm-core/pom.xml | 2 +- storm-dist/binary/pom.xml | 2 +- storm-dist/source/pom.xml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index a274e3f103..7a3a840e40 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ org.apache.storm storm - 0.9.1-incubating + 0.9.2-incubating-SNAPSHOT pom Storm Distributed and fault-tolerant realtime computation @@ -135,7 +135,7 @@ scm:git:https://git-wip-us.apache.org/repos/asf/incubator-storm.git scm:git:https://git-wip-us.apache.org/repos/asf/incubator-storm.git - v0.9.1-incubating + HEAD https://git-wip-us.apache.org/repos/asf/incubator-storm diff --git a/storm-buildtools/maven-shade-clojure-transformer/pom.xml b/storm-buildtools/maven-shade-clojure-transformer/pom.xml index bfaafc097f..7dfa2a2051 100644 --- a/storm-buildtools/maven-shade-clojure-transformer/pom.xml +++ b/storm-buildtools/maven-shade-clojure-transformer/pom.xml @@ -21,7 +21,7 @@ storm org.apache.storm - 0.9.1-incubating + 0.9.2-incubating-SNAPSHOT ../../pom.xml diff --git a/storm-core/pom.xml b/storm-core/pom.xml index af3a338d97..777e8d5e6c 100644 --- a/storm-core/pom.xml +++ b/storm-core/pom.xml @@ -20,7 +20,7 @@ storm org.apache.storm - 0.9.1-incubating + 0.9.2-incubating-SNAPSHOT org.apache.storm storm-core diff --git a/storm-dist/binary/pom.xml b/storm-dist/binary/pom.xml index 15b76b2aa0..31207094a9 100644 --- a/storm-dist/binary/pom.xml +++ b/storm-dist/binary/pom.xml @@ -21,7 +21,7 @@ storm org.apache.storm - 0.9.1-incubating + 0.9.2-incubating-SNAPSHOT ../../pom.xml org.apache.storm diff --git a/storm-dist/source/pom.xml b/storm-dist/source/pom.xml index aa6c13859f..83300946da 100644 --- a/storm-dist/source/pom.xml +++ b/storm-dist/source/pom.xml @@ -21,7 +21,7 @@ storm org.apache.storm - 0.9.1-incubating + 0.9.2-incubating-SNAPSHOT ../../pom.xml org.apache.storm From b6b1f1d8fb5a56cf14c44ff46aa513e0fa4e99ab Mon Sep 17 00:00:00 2001 From: Danijel Schiavuzzi Date: Tue, 11 Feb 2014 15:28:59 +0100 Subject: [PATCH 269/349] Make the spout's Storm Metrics' time bucket size configurable via TridentKafkaConfig. Leave the default time bucket size at 60 seconds. --- pom.xml | 4 ++-- src/jvm/storm/kafka/KafkaConfig.java | 1 + src/jvm/storm/kafka/trident/TridentKafkaEmitter.java | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 910041ad90..8dbd9a9147 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ net.wurstmeister.storm storm-kafka-0.8-plus jar - 0.4.0-SNAPSHOT + 0.4.0-configurable-metrics-emit-interval-SNAPSHOT storm-kafka-0.8-plus Storm module for kafka > 0.8 @@ -170,7 +170,7 @@ storm storm - 0.9.0 + 0.9.0.1 provided diff --git a/src/jvm/storm/kafka/KafkaConfig.java b/src/jvm/storm/kafka/KafkaConfig.java index dddcead1bd..8ef2a88b7a 100644 --- a/src/jvm/storm/kafka/KafkaConfig.java +++ b/src/jvm/storm/kafka/KafkaConfig.java @@ -18,6 +18,7 @@ public class KafkaConfig implements Serializable { public boolean forceFromStart = false; public long startOffsetTime = kafka.api.OffsetRequest.EarliestTime(); public boolean useStartOffsetTimeIfOffsetOutOfRange = true; + public int metricsTimeBucketSizeInSecs = 60; public KafkaConfig(BrokerHosts hosts, String topic) { this(hosts, topic, kafka.api.OffsetRequest.DefaultClientId()); diff --git a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java index fbbbd4bdce..66785f07dc 100644 --- a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java +++ b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java @@ -49,9 +49,9 @@ public TridentKafkaEmitter(Map conf, TopologyContext context, TridentKafkaConfig _connections = new DynamicPartitionConnections(_config, KafkaUtils.makeBrokerReader(conf, _config)); _topologyName = (String) conf.get(Config.TOPOLOGY_NAME); _kafkaOffsetMetric = new KafkaUtils.KafkaOffsetMetric(_config.topic, _connections); - context.registerMetric("kafkaOffset", _kafkaOffsetMetric, 60); - _kafkaMeanFetchLatencyMetric = context.registerMetric("kafkaFetchAvg", new MeanReducer(), 60); - _kafkaMaxFetchLatencyMetric = context.registerMetric("kafkaFetchMax", new MaxMetric(), 60); + context.registerMetric("kafkaOffset", _kafkaOffsetMetric, _config.metricsTimeBucketSizeInSecs); + _kafkaMeanFetchLatencyMetric = context.registerMetric("kafkaFetchAvg", new MeanReducer(), _config.metricsTimeBucketSizeInSecs); + _kafkaMaxFetchLatencyMetric = context.registerMetric("kafkaFetchMax", new MaxMetric(), _config.metricsTimeBucketSizeInSecs); } From fe1a9da07d7d075a9db768d5fa332f1d24bc3a37 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Sat, 15 Feb 2014 14:12:36 -0500 Subject: [PATCH 270/349] remove Nathan Marz reference from NOTICE file --- NOTICE | 7 +------ storm-dist/binary/NOTICE | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/NOTICE b/NOTICE index 4f3e648d8e..35e9475a71 100644 --- a/NOTICE +++ b/NOTICE @@ -1,13 +1,8 @@ Apache Storm -Copyright 2013 The Apache Software Foundation +Copyright 2014 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). - -This product includes software developed by Nathan Marz -Copyright 2011-2013 Nathan Marz - - This product includes software developed by Yahoo! Inc. (www.yahoo.com) Copyright © 2012-2013 Yahoo! Inc. All rights reserved. diff --git a/storm-dist/binary/NOTICE b/storm-dist/binary/NOTICE index 0eec4766f8..43fcf84020 100644 --- a/storm-dist/binary/NOTICE +++ b/storm-dist/binary/NOTICE @@ -1,14 +1,9 @@ Apache Storm -Copyright 2013 The Apache Software Foundation +Copyright 2014 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). - -This product includes software developed by Nathan Marz -Copyright 2011-2013 Nathan Marz - - This product includes software developed by Yahoo! Inc. (www.yahoo.com) Copyright © 2012-2013 Yahoo! Inc. All rights reserved. From 254ec135b9a67b1e7bc8e979356274aee2e7d715 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Sat, 15 Feb 2014 14:24:51 -0500 Subject: [PATCH 271/349] undo last commit --- NOTICE | 4 ++++ storm-dist/binary/NOTICE | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/NOTICE b/NOTICE index 35e9475a71..5bdb68ccf9 100644 --- a/NOTICE +++ b/NOTICE @@ -4,5 +4,9 @@ Copyright 2014 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). +This product includes software developed by Nathan Marz +Copyright 2011-2013 Nathan Marz + + This product includes software developed by Yahoo! Inc. (www.yahoo.com) Copyright © 2012-2013 Yahoo! Inc. All rights reserved. diff --git a/storm-dist/binary/NOTICE b/storm-dist/binary/NOTICE index 43fcf84020..0eec4766f8 100644 --- a/storm-dist/binary/NOTICE +++ b/storm-dist/binary/NOTICE @@ -1,9 +1,14 @@ Apache Storm -Copyright 2014 The Apache Software Foundation +Copyright 2013 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). + +This product includes software developed by Nathan Marz +Copyright 2011-2013 Nathan Marz + + This product includes software developed by Yahoo! Inc. (www.yahoo.com) Copyright © 2012-2013 Yahoo! Inc. All rights reserved. From db61558bb480dd6f39eaa3e6235d9f0c193f3a6a Mon Sep 17 00:00:00 2001 From: anfeng Date: Sat, 15 Feb 2014 11:55:42 -0800 Subject: [PATCH 272/349] remove Yahoo copyright from NOTICE files --- NOTICE | 5 +---- storm-dist/binary/NOTICE | 3 --- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/NOTICE b/NOTICE index 35e9475a71..a6a160ab55 100644 --- a/NOTICE +++ b/NOTICE @@ -2,7 +2,4 @@ Apache Storm Copyright 2014 The Apache Software Foundation This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This product includes software developed by Yahoo! Inc. (www.yahoo.com) -Copyright © 2012-2013 Yahoo! Inc. All rights reserved. +The Apache Software Foundation (http://www.apache.org/). \ No newline at end of file diff --git a/storm-dist/binary/NOTICE b/storm-dist/binary/NOTICE index 43fcf84020..2036617b73 100644 --- a/storm-dist/binary/NOTICE +++ b/storm-dist/binary/NOTICE @@ -4,9 +4,6 @@ Copyright 2014 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). -This product includes software developed by Yahoo! Inc. (www.yahoo.com) -Copyright © 2012-2013 Yahoo! Inc. All rights reserved. - YAML support provided by snakeyaml (http://code.google.com/p/snakeyaml/). Copyright (c) 2008-2010 Andrey Somov From 021094131c13909bd9195cb3e3b9c9775ee2fcf5 Mon Sep 17 00:00:00 2001 From: supercargo Date: Sun, 16 Feb 2014 08:23:46 -0500 Subject: [PATCH 273/349] STORM-241: making thrift binary chunk size configurable --- conf/defaults.yaml | 1 + storm-core/src/jvm/backtype/storm/Config.java | 6 ++++++ storm-core/src/jvm/backtype/storm/StormSubmitter.java | 5 ++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/conf/defaults.yaml b/conf/defaults.yaml index 2dbba24c9f..d0b87a053e 100644 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@ -35,6 +35,7 @@ storm.zookeeper.retry.intervalceiling.millis: 30000 storm.cluster.mode: "distributed" # can be distributed or local storm.local.mode.zmq: false storm.thrift.transport: "backtype.storm.security.auth.SimpleTransportPlugin" +storm.thrift.chunk.size: 307200 storm.messaging.transport: "backtype.storm.messaging.netty.Context" ### nimbus.* configs are for the master diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java index 281ae525df..1a308c283f 100644 --- a/storm-core/src/jvm/backtype/storm/Config.java +++ b/storm-core/src/jvm/backtype/storm/Config.java @@ -136,6 +136,12 @@ public class Config extends HashMap { public static final String STORM_THRIFT_TRANSPORT_PLUGIN = "storm.thrift.transport"; public static final Object STORM_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class; + /** + * The chunk size in bytes to use when sending binary blobs over the Thrift protocol + */ + public static final String STORM_THRIFT_CHUNK_SIZE = "storm.thrift.chunk.size"; + public static final Object STORM_THRIFT_CHUNK_SIZE_SCHEMA = Number.class; + /** * The serializer class for ListDelegate (tuple payload). * The default serializer will be ListDelegateSerializer diff --git a/storm-core/src/jvm/backtype/storm/StormSubmitter.java b/storm-core/src/jvm/backtype/storm/StormSubmitter.java index 80734568b0..b5081e8d44 100644 --- a/storm-core/src/jvm/backtype/storm/StormSubmitter.java +++ b/storm-core/src/jvm/backtype/storm/StormSubmitter.java @@ -148,10 +148,13 @@ public static String submitJar(Map conf, String localJar) { throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload."); } NimbusClient client = NimbusClient.getConfiguredClient(conf); + + int thriftChunkSize = ((Number) conf.get(Config.STORM_THRIFT_CHUNK_SIZE)).intValue(); + try { String uploadLocation = client.getClient().beginFileUpload(); LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation); - BufferFileInputStream is = new BufferFileInputStream(localJar); + BufferFileInputStream is = new BufferFileInputStream(localJar, thriftChunkSize); while(true) { byte[] toSubmit = is.read(); if(toSubmit.length==0) break; From 2a0757ace47466e4ce13f670e977128c546c0349 Mon Sep 17 00:00:00 2001 From: Danijel Schiavuzzi Date: Mon, 17 Feb 2014 11:43:20 +0100 Subject: [PATCH 274/349] Update pom.xml Revert custom changes --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 8dbd9a9147..910041ad90 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ net.wurstmeister.storm storm-kafka-0.8-plus jar - 0.4.0-configurable-metrics-emit-interval-SNAPSHOT + 0.4.0-SNAPSHOT storm-kafka-0.8-plus Storm module for kafka > 0.8 @@ -170,7 +170,7 @@ storm storm - 0.9.0.1 + 0.9.0 provided From d2a0bcd99e806922fb6443d2764f8c3ceb52aca1 Mon Sep 17 00:00:00 2001 From: anfeng Date: Mon, 17 Feb 2014 19:01:23 -0800 Subject: [PATCH 275/349] NOTICE revised regarding Yahoo contribution --- NOTICE | 5 ++++- storm-dist/binary/NOTICE | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/NOTICE b/NOTICE index a6a160ab55..637e578c5e 100644 --- a/NOTICE +++ b/NOTICE @@ -2,4 +2,7 @@ Apache Storm Copyright 2014 The Apache Software Foundation This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). \ No newline at end of file +The Apache Software Foundation (http://www.apache.org/). + +This product includes software developed by Yahoo! Inc. (www.yahoo.com) +Copyright (c) 2012-2014 Yahoo! Inc. \ No newline at end of file diff --git a/storm-dist/binary/NOTICE b/storm-dist/binary/NOTICE index 2036617b73..6e3d43b5dd 100644 --- a/storm-dist/binary/NOTICE +++ b/storm-dist/binary/NOTICE @@ -4,6 +4,9 @@ Copyright 2014 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). +This product includes software developed by Yahoo! Inc. (www.yahoo.com) +Copyright (c) 2012-2014 Yahoo! Inc. + YAML support provided by snakeyaml (http://code.google.com/p/snakeyaml/). Copyright (c) 2008-2010 Andrey Somov From 15c489018c0cc7a32cc8e5c157232da4d1e26f2d Mon Sep 17 00:00:00 2001 From: Kang Xiao Date: Wed, 19 Feb 2014 00:22:01 +0800 Subject: [PATCH 276/349] STORM-186 fix float secs to millis long convertion --- storm-core/src/clj/backtype/storm/timer.clj | 2 +- storm-core/src/clj/backtype/storm/util.clj | 3 +++ storm-core/test/clj/backtype/storm/utils_test.clj | 11 +++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/storm-core/src/clj/backtype/storm/timer.clj b/storm-core/src/clj/backtype/storm/timer.clj index f9774b6d1a..78b1f1c246 100644 --- a/storm-core/src/clj/backtype/storm/timer.clj +++ b/storm-core/src/clj/backtype/storm/timer.clj @@ -82,7 +82,7 @@ (let [id (uuid) ^PriorityQueue queue (:queue timer)] (locking (:lock timer) - (.add queue [(+ (current-time-millis) (* 1000 (long delay-secs))) afn id]) + (.add queue [(+ (current-time-millis) (secs-to-millis-long delay-secs)) afn id]) ))) (defn schedule-recurring [timer delay-secs recur-secs afn] diff --git a/storm-core/src/clj/backtype/storm/util.clj b/storm-core/src/clj/backtype/storm/util.clj index 2ab30c7e24..a8f238a682 100644 --- a/storm-core/src/clj/backtype/storm/util.clj +++ b/storm-core/src/clj/backtype/storm/util.clj @@ -220,6 +220,9 @@ (defn current-time-millis [] (Time/currentTimeMillis)) +(defn secs-to-millis-long [secs] + (long (* 1000 secs))) + (defn clojurify-structure [s] (prewalk (fn [x] (cond (instance? Map x) (into {} x) diff --git a/storm-core/test/clj/backtype/storm/utils_test.clj b/storm-core/test/clj/backtype/storm/utils_test.clj index 99bf7d113f..b24db623d0 100644 --- a/storm-core/test/clj/backtype/storm/utils_test.clj +++ b/storm-core/test/clj/backtype/storm/utils_test.clj @@ -62,3 +62,14 @@ )) ) ) + +(deftest test-secs-to-millis-long + (is (= 0 (secs-to-millis-long 0))) + (is (= 2 (secs-to-millis-long 0.002))) + (is (= 500 (secs-to-millis-long 0.5))) + (is (= 1000 (secs-to-millis-long 1))) + (is (= 1080 (secs-to-millis-long 1.08))) + (is (= 10000 (secs-to-millis-long 10))) + (is (= 10100 (secs-to-millis-long 10.1))) +) + From f8afa99c3036918291c694163e81fd4f22e0460b Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Wed, 19 Feb 2014 22:55:45 +0000 Subject: [PATCH 277/349] use metricsTimeBucketSizeInSecs consistently --- src/jvm/storm/kafka/KafkaSpout.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jvm/storm/kafka/KafkaSpout.java b/src/jvm/storm/kafka/KafkaSpout.java index d0975106c8..79e33fe833 100644 --- a/src/jvm/storm/kafka/KafkaSpout.java +++ b/src/jvm/storm/kafka/KafkaSpout.java @@ -93,7 +93,7 @@ public Object getValueAndReset() { } return _kafkaOffsetMetric.getValueAndReset(); } - }, 60); + }, _spoutConfig.metricsTimeBucketSizeInSecs); context.registerMetric("kafkaPartition", new IMetric() { @Override @@ -105,7 +105,7 @@ public Object getValueAndReset() { } return concatMetricsDataMaps; } - }, 60); + }, _spoutConfig.metricsTimeBucketSizeInSecs); } @Override From 7ed8879a1fdb27df6848bf2921f8cef89408f2ef Mon Sep 17 00:00:00 2001 From: Kang Xiao Date: Thu, 20 Feb 2014 12:49:45 +0800 Subject: [PATCH 278/349] STORM-186 convert 1000 to long to avoid int overflow --- storm-core/src/clj/backtype/storm/util.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storm-core/src/clj/backtype/storm/util.clj b/storm-core/src/clj/backtype/storm/util.clj index a8f238a682..580e9500c3 100644 --- a/storm-core/src/clj/backtype/storm/util.clj +++ b/storm-core/src/clj/backtype/storm/util.clj @@ -221,7 +221,7 @@ (Time/currentTimeMillis)) (defn secs-to-millis-long [secs] - (long (* 1000 secs))) + (long (* (long 1000) secs))) (defn clojurify-structure [s] (prewalk (fn [x] From 4de85c8e7f6f4244be9eb688c6a4576a9c88c2a1 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 23 Feb 2014 12:48:56 +0000 Subject: [PATCH 279/349] Added support for reading message key --- src/jvm/storm/kafka/KafkaUtils.java | 22 +++++- src/jvm/storm/kafka/KeyValueScheme.java | 11 +++ .../kafka/KeyValueSchemeAsMultiScheme.java | 19 +++++ src/jvm/storm/kafka/PartitionManager.java | 2 +- src/jvm/storm/kafka/StringKeyValueScheme.java | 20 +++++ src/jvm/storm/kafka/StringScheme.java | 10 ++- .../kafka/trident/TridentKafkaEmitter.java | 3 +- src/test/storm/kafka/KafkaUtilsTest.java | 76 +++++++++++++++++-- .../storm/kafka/StringKeyValueSchemeTest.java | 38 ++++++++++ 9 files changed, 187 insertions(+), 14 deletions(-) create mode 100644 src/jvm/storm/kafka/KeyValueScheme.java create mode 100644 src/jvm/storm/kafka/KeyValueSchemeAsMultiScheme.java create mode 100644 src/jvm/storm/kafka/StringKeyValueScheme.java create mode 100644 src/test/storm/kafka/StringKeyValueSchemeTest.java diff --git a/src/jvm/storm/kafka/KafkaUtils.java b/src/jvm/storm/kafka/KafkaUtils.java index d86519c524..300d998404 100644 --- a/src/jvm/storm/kafka/KafkaUtils.java +++ b/src/jvm/storm/kafka/KafkaUtils.java @@ -1,6 +1,7 @@ package storm.kafka; import backtype.storm.metric.api.IMetric; +import backtype.storm.utils.Utils; import kafka.api.FetchRequest; import kafka.api.FetchRequestBuilder; import kafka.api.PartitionOffsetRequestInfo; @@ -9,6 +10,7 @@ import kafka.javaapi.OffsetRequest; import kafka.javaapi.consumer.SimpleConsumer; import kafka.javaapi.message.ByteBufferMessageSet; +import kafka.message.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import storm.kafka.trident.IBrokerReader; @@ -16,10 +18,8 @@ import storm.kafka.trident.ZkBrokerReader; import java.net.ConnectException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; +import java.nio.ByteBuffer; +import java.util.*; public class KafkaUtils { @@ -165,4 +165,18 @@ public static ByteBufferMessageSet fetchMessages(KafkaConfig config, SimpleConsu } return msgs; } + + + public static Iterable> generateTuples(KafkaConfig kafkaConfig, Message msg) { + Iterable> tups; + ByteBuffer payload = msg.payload(); + ByteBuffer key = msg.key(); + if (key != null && kafkaConfig.scheme instanceof KeyValueSchemeAsMultiScheme) { + tups = ((KeyValueSchemeAsMultiScheme) kafkaConfig.scheme).deserializeKeyAndValue(Utils.toByteArray(key), Utils.toByteArray(payload)); + } else { + tups = kafkaConfig.scheme.deserialize(Utils.toByteArray(payload)); + } + return tups; + } + } diff --git a/src/jvm/storm/kafka/KeyValueScheme.java b/src/jvm/storm/kafka/KeyValueScheme.java new file mode 100644 index 0000000000..df31cb8b3b --- /dev/null +++ b/src/jvm/storm/kafka/KeyValueScheme.java @@ -0,0 +1,11 @@ +package storm.kafka; + +import backtype.storm.spout.Scheme; + +import java.util.List; + +public interface KeyValueScheme extends Scheme { + + public List deserializeKeyAndValue(byte[] key, byte[] value); + +} diff --git a/src/jvm/storm/kafka/KeyValueSchemeAsMultiScheme.java b/src/jvm/storm/kafka/KeyValueSchemeAsMultiScheme.java new file mode 100644 index 0000000000..2412a1ce78 --- /dev/null +++ b/src/jvm/storm/kafka/KeyValueSchemeAsMultiScheme.java @@ -0,0 +1,19 @@ +package storm.kafka; + +import backtype.storm.spout.SchemeAsMultiScheme; +import java.util.Arrays; +import java.util.List; + +public class KeyValueSchemeAsMultiScheme extends SchemeAsMultiScheme{ + + public KeyValueSchemeAsMultiScheme(KeyValueScheme scheme) { + super(scheme); + } + + public Iterable> deserializeKeyAndValue(final byte[] key, final byte[] value) { + List o = ((KeyValueScheme)scheme).deserializeKeyAndValue(key, value); + if(o == null) return null; + else return Arrays.asList(o); + } + +} diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index f12c0d991e..fc9d817189 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -112,7 +112,7 @@ public EmitState next(SpoutOutputCollector collector) { if (toEmit == null) { return EmitState.NO_EMITTED; } - Iterable> tups = _spoutConfig.scheme.deserialize(Utils.toByteArray(toEmit.msg.payload())); + Iterable> tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.msg); if (tups != null) { for (List tup : tups) { collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset)); diff --git a/src/jvm/storm/kafka/StringKeyValueScheme.java b/src/jvm/storm/kafka/StringKeyValueScheme.java new file mode 100644 index 0000000000..a6adddb95c --- /dev/null +++ b/src/jvm/storm/kafka/StringKeyValueScheme.java @@ -0,0 +1,20 @@ +package storm.kafka; + +import backtype.storm.tuple.Values; +import com.google.common.collect.ImmutableMap; + +import java.util.List; + +public class StringKeyValueScheme extends StringScheme implements KeyValueScheme { + + @Override + public List deserializeKeyAndValue(byte[] key, byte[] value) { + if ( key == null ) { + return deserialize(value); + } + String keyString = StringScheme.deserializeString(key); + String valueString = StringScheme.deserializeString(value); + return new Values(ImmutableMap.of(keyString, valueString)); + } + +} diff --git a/src/jvm/storm/kafka/StringScheme.java b/src/jvm/storm/kafka/StringScheme.java index c4d8270e2d..a8094484ef 100644 --- a/src/jvm/storm/kafka/StringScheme.java +++ b/src/jvm/storm/kafka/StringScheme.java @@ -9,15 +9,21 @@ public class StringScheme implements Scheme { + public static final String STRING_SCHEME_KEY = "str"; + public List deserialize(byte[] bytes) { + return new Values(deserializeString(bytes)); + } + + public static String deserializeString(byte[] string) { try { - return new Values(new String(bytes, "UTF-8")); + return new String(string, "UTF-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } } public Fields getOutputFields() { - return new Fields("str"); + return new Fields(STRING_SCHEME_KEY); } } \ No newline at end of file diff --git a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java index 66785f07dc..973ce8f7b9 100644 --- a/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java +++ b/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java @@ -155,8 +155,7 @@ private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector c } private void emit(TridentCollector collector, Message msg) { - Iterable> values = - _config.scheme.deserialize(Utils.toByteArray(msg.payload())); + Iterable> values = KafkaUtils.generateTuples(_config, msg); if (values != null) { for (List value : values) { collector.emit(value); diff --git a/src/test/storm/kafka/KafkaUtilsTest.java b/src/test/storm/kafka/KafkaUtilsTest.java index 20a4221644..db270c26ea 100644 --- a/src/test/storm/kafka/KafkaUtilsTest.java +++ b/src/test/storm/kafka/KafkaUtilsTest.java @@ -1,10 +1,13 @@ package storm.kafka; +import backtype.storm.spout.SchemeAsMultiScheme; import backtype.storm.utils.Utils; +import com.google.common.collect.ImmutableMap; import kafka.api.OffsetRequest; import kafka.javaapi.consumer.SimpleConsumer; import kafka.javaapi.message.ByteBufferMessageSet; import kafka.javaapi.producer.Producer; +import kafka.message.MessageAndOffset; import kafka.producer.KeyedMessage; import kafka.producer.ProducerConfig; import org.junit.After; @@ -12,10 +15,12 @@ import org.junit.Test; import storm.kafka.trident.GlobalPartitionInformation; +import java.util.List; import java.util.Properties; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; public class KafkaUtilsTest { @@ -90,19 +95,80 @@ public void getOffsetFromConfigAndFroceFromStart() { assertThat(earliestOffset, is(equalTo(offsetFromConfig))); } - private String createTopicAndSendMessage() { + @Test + public void generateTuplesWithoutKeyAndKeyValueScheme() { + config.scheme = new KeyValueSchemeAsMultiScheme(new StringKeyValueScheme()); + runGetValueOnlyTuplesTest(); + } + + @Test + public void generateTuplesWithKeyAndKeyValueScheme() { + config.scheme = new KeyValueSchemeAsMultiScheme(new StringKeyValueScheme()); + String value = "value"; + String key = "key"; + createTopicAndSendMessage(key, value); + ByteBufferMessageSet messageAndOffsets = getLastMessage(); + for (MessageAndOffset msg : messageAndOffsets) { + Iterable> lists = KafkaUtils.generateTuples(config, msg.message()); + assertEquals(ImmutableMap.of(key, value), lists.iterator().next().get(0)); + } + } + + @Test + public void generateTupelsWithValueScheme() { + config.scheme = new SchemeAsMultiScheme(new StringScheme()); + runGetValueOnlyTuplesTest(); + } + + @Test + public void generateTuplesWithValueSchemeAndKeyValueMessage() { + config.scheme = new SchemeAsMultiScheme(new StringScheme()); + String value = "value"; + String key = "key"; + createTopicAndSendMessage(key, value); + ByteBufferMessageSet messageAndOffsets = getLastMessage(); + for (MessageAndOffset msg : messageAndOffsets) { + Iterable> lists = KafkaUtils.generateTuples(config, msg.message()); + assertEquals(value, lists.iterator().next().get(0)); + } + } + + private ByteBufferMessageSet getLastMessage() { + long offsetOfLastMessage = KafkaUtils.getOffset(simpleConsumer, config.topic, 0, OffsetRequest.LatestTime()) - 1; + return KafkaUtils.fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), offsetOfLastMessage); + } + + private void runGetValueOnlyTuplesTest() { + String value = "value"; + createTopicAndSendMessage(null, value); + ByteBufferMessageSet messageAndOffsets = getLastMessage(); + for (MessageAndOffset msg : messageAndOffsets) { + Iterable> lists = KafkaUtils.generateTuples(config, msg.message()); + assertEquals(value, lists.iterator().next().get(0)); + } + } + + + private void createTopicAndSendMessage() { + createTopicAndSendMessage(null, "someValue"); + } + + private void createTopicAndSendMessage(String value) { + createTopicAndSendMessage(null, value); + } + + private void createTopicAndSendMessage(String key, String value) { Properties p = new Properties(); p.setProperty("metadata.broker.list", "localhost:49123"); p.setProperty("serializer.class", "kafka.serializer.StringEncoder"); ProducerConfig producerConfig = new ProducerConfig(p); Producer producer = new Producer(producerConfig); - String value = "value"; - producer.send(new KeyedMessage(config.topic, value)); - return value; + producer.send(new KeyedMessage(config.topic, key, value)); } private void sendMessageAndAssertValueForOffset(long offset) { - String value = createTopicAndSendMessage(); + String value = "test"; + createTopicAndSendMessage(value); ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), offset); String message = new String(Utils.toByteArray(messageAndOffsets.iterator().next().message().payload())); assertThat(message, is(equalTo(value))); diff --git a/src/test/storm/kafka/StringKeyValueSchemeTest.java b/src/test/storm/kafka/StringKeyValueSchemeTest.java new file mode 100644 index 0000000000..4413c7bd74 --- /dev/null +++ b/src/test/storm/kafka/StringKeyValueSchemeTest.java @@ -0,0 +1,38 @@ +package storm.kafka; + +import backtype.storm.tuple.Fields; +import com.google.common.collect.ImmutableMap; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class StringKeyValueSchemeTest { + + private StringKeyValueScheme scheme = new StringKeyValueScheme(); + + @Test + public void testDeserialize() throws Exception { + assertEquals(Arrays.asList("test"), scheme.deserialize("test".getBytes())); + } + + @Test + public void testGetOutputFields() throws Exception { + Fields outputFields = scheme.getOutputFields(); + assertTrue(outputFields.contains(StringScheme.STRING_SCHEME_KEY)); + assertEquals(1, outputFields.size()); + } + + @Test + public void testDeserializeWithNullKeyAndValue() throws Exception { + assertEquals(Arrays.asList("test"), scheme.deserializeKeyAndValue(null, "test".getBytes())); + } + + @Test + public void testDeserializeWithKeyAndValue() throws Exception { + assertEquals(Arrays.asList(ImmutableMap.of("key", "test")), + scheme.deserializeKeyAndValue("key".getBytes(), "test".getBytes())); + } +} From bd0cc453a5d7389eb2a92bacef46c13b436a0316 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sun, 23 Feb 2014 16:10:38 +0000 Subject: [PATCH 280/349] fixed tests --- src/test/storm/kafka/KafkaUtilsTest.java | 31 ++++++++++++++---------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/test/storm/kafka/KafkaUtilsTest.java b/src/test/storm/kafka/KafkaUtilsTest.java index db270c26ea..07630427cf 100644 --- a/src/test/storm/kafka/KafkaUtilsTest.java +++ b/src/test/storm/kafka/KafkaUtilsTest.java @@ -28,13 +28,14 @@ public class KafkaUtilsTest { private KafkaTestBroker broker; private SimpleConsumer simpleConsumer; private KafkaConfig config; + private BrokerHosts brokerHosts; @Before public void setup() { broker = new KafkaTestBroker(); GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(); globalPartitionInformation.addPartition(0, Broker.fromString(broker.getBrokerConnectionString())); - BrokerHosts brokerHosts = new StaticHosts(globalPartitionInformation); + brokerHosts = new StaticHosts(globalPartitionInformation); config = new KafkaConfig(brokerHosts, "testTopic"); simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient"); } @@ -60,19 +61,31 @@ public void brokerIsDown() throws Exception { @Test public void fetchMessage() throws Exception { - long lastOffset = KafkaUtils.getOffset(simpleConsumer, config.topic, 0, OffsetRequest.EarliestTime()); - sendMessageAndAssertValueForOffset(lastOffset); + String value = "test"; + createTopicAndSendMessage(value); + long offset = KafkaUtils.getOffset(simpleConsumer, config.topic, 0, OffsetRequest.LatestTime()) - 1; + ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(config, simpleConsumer, + new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), offset); + String message = new String(Utils.toByteArray(messageAndOffsets.iterator().next().message().payload())); + assertThat(message, is(equalTo(value))); } @Test(expected = FailedFetchException.class) public void fetchMessagesWithInvalidOffsetAndDefaultHandlingDisabled() throws Exception { config.useStartOffsetTimeIfOffsetOutOfRange = false; - sendMessageAndAssertValueForOffset(-99); + KafkaUtils.fetchMessages(config, simpleConsumer, + new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), -99); } @Test public void fetchMessagesWithInvalidOffsetAndDefaultHandlingEnabled() throws Exception { - sendMessageAndAssertValueForOffset(-99); + config = new KafkaConfig(brokerHosts, "newTopic"); + String value = "test"; + createTopicAndSendMessage(value); + ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(config, simpleConsumer, + new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), -99); + String message = new String(Utils.toByteArray(messageAndOffsets.iterator().next().message().payload())); + assertThat(message, is(equalTo(value))); } @Test @@ -165,12 +178,4 @@ private void createTopicAndSendMessage(String key, String value) { Producer producer = new Producer(producerConfig); producer.send(new KeyedMessage(config.topic, key, value)); } - - private void sendMessageAndAssertValueForOffset(long offset) { - String value = "test"; - createTopicAndSendMessage(value); - ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), offset); - String message = new String(Utils.toByteArray(messageAndOffsets.iterator().next().message().payload())); - assertThat(message, is(equalTo(value))); - } } From c695c1b06cd1f44fbbe1181a41a6c4ffe2186e01 Mon Sep 17 00:00:00 2001 From: Danijel Schiavuzzi Date: Thu, 20 Feb 2014 18:09:25 +0100 Subject: [PATCH 281/349] Add a new per-partition and total metric, "earliestTimeOffset". Rename "latestTime" metric to "latestTimeOffset" to more closely conform to other, related metrics' nomenclature. --- src/jvm/storm/kafka/KafkaUtils.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/jvm/storm/kafka/KafkaUtils.java b/src/jvm/storm/kafka/KafkaUtils.java index 300d998404..eed438f5dc 100644 --- a/src/jvm/storm/kafka/KafkaUtils.java +++ b/src/jvm/storm/kafka/KafkaUtils.java @@ -79,6 +79,7 @@ public void setLatestEmittedOffset(Partition partition, long offset) { public Object getValueAndReset() { try { long totalSpoutLag = 0; + long totalEarliestTimeOffset = 0; long totalLatestTimeOffset = 0; long totalLatestEmittedOffset = 0; HashMap ret = new HashMap(); @@ -90,22 +91,26 @@ public Object getValueAndReset() { LOG.warn("partitionToOffset contains partition not found in _connections. Stale partition data?"); return null; } + long earliestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.EarliestTime()); long latestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.LatestTime()); - if (latestTimeOffset == 0) { + if (earliestTimeOffset == 0 || latestTimeOffset == 0) { LOG.warn("No data found in Kafka Partition " + partition.getId()); return null; } long latestEmittedOffset = e.getValue(); long spoutLag = latestTimeOffset - latestEmittedOffset; ret.put(partition.getId() + "/" + "spoutLag", spoutLag); - ret.put(partition.getId() + "/" + "latestTime", latestTimeOffset); + ret.put(partition.getId() + "/" + "earliestTimeOffset", earliestTimeOffset); + ret.put(partition.getId() + "/" + "latestTimeOffset", latestTimeOffset); ret.put(partition.getId() + "/" + "latestEmittedOffset", latestEmittedOffset); totalSpoutLag += spoutLag; + totalEarliestTimeOffset += earliestTimeOffset; totalLatestTimeOffset += latestTimeOffset; totalLatestEmittedOffset += latestEmittedOffset; } ret.put("totalSpoutLag", totalSpoutLag); - ret.put("totalLatestTime", totalLatestTimeOffset); + ret.put("totalEarliestTimeOffset", totalEarliestTimeOffset); + ret.put("totalLatestTimeOffset", totalLatestTimeOffset); ret.put("totalLatestEmittedOffset", totalLatestEmittedOffset); return ret; } else { From f573001db106d6b8d6af2167bb2b88f38a3738e5 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Wed, 26 Feb 2014 22:08:36 +0000 Subject: [PATCH 282/349] preparing for tag --- CHANGELOG.md | 4 ++++ pom.xml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ced0ffc975..74ab824c12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.4.0 +* added support for reading kafka message keys +* configurable metrics emit interval ## 0.3.0 * updated partition path in zookeeper * added error handling for fetch request + diff --git a/pom.xml b/pom.xml index 910041ad90..c542158f91 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ net.wurstmeister.storm storm-kafka-0.8-plus jar - 0.4.0-SNAPSHOT + 0.4.0 storm-kafka-0.8-plus Storm module for kafka > 0.8 From 118cec43b19eec2ed1d66f6dd0aac7209edfde96 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Wed, 26 Feb 2014 22:38:21 +0000 Subject: [PATCH 283/349] added Kafka bolt --- pom.xml | 8 +- src/jvm/storm/kafka/bolt/KafkaBolt.java | 72 ++++++++ src/test/storm/kafka/bolt/KafkaBoltTest.java | 170 +++++++++++++++++++ 3 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 src/jvm/storm/kafka/bolt/KafkaBolt.java create mode 100644 src/test/storm/kafka/bolt/KafkaBoltTest.java diff --git a/pom.xml b/pom.xml index c542158f91..ff9b9c1a38 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ net.wurstmeister.storm storm-kafka-0.8-plus jar - 0.4.0 + 0.5.0-SNAPSHOT storm-kafka-0.8-plus Storm module for kafka > 0.8 @@ -98,6 +98,12 @@ + + org.mockito + mockito-all + 1.9.0 + test + org.scala-lang scala-library diff --git a/src/jvm/storm/kafka/bolt/KafkaBolt.java b/src/jvm/storm/kafka/bolt/KafkaBolt.java new file mode 100644 index 0000000000..89969d9a92 --- /dev/null +++ b/src/jvm/storm/kafka/bolt/KafkaBolt.java @@ -0,0 +1,72 @@ +package storm.kafka.bolt; + +import backtype.storm.task.OutputCollector; +import backtype.storm.task.TopologyContext; +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.topology.base.BaseRichBolt; +import backtype.storm.tuple.Tuple; +import kafka.javaapi.producer.Producer; +import kafka.producer.KeyedMessage; +import kafka.producer.ProducerConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Properties; + + +/** + * Bolt implementation that can send Tuple data to Kafka + *

+ * It expects the producer configuration and topic in storm config under + *

+ * 'kafka.broker.properties' and 'topic' + *

+ * respectively. + */ +public class KafkaBolt extends BaseRichBolt { + + private static final Logger LOG = LoggerFactory.getLogger(KafkaBolt.class); + + public static final String TOPIC = "topic"; + public static final String KAFKA_BROKER_PROPERTIES = "kafka.broker.properties"; + + public static final String BOLT_KEY = "key"; + public static final String BOLT_MESSAGE = "message"; + + private Producer producer; + private OutputCollector collector; + private String topic; + + @Override + public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { + Map configMap = (Map) stormConf.get(KAFKA_BROKER_PROPERTIES); + Properties properties = new Properties(); + properties.putAll(configMap); + ProducerConfig config = new ProducerConfig(properties); + producer = new Producer(config); + this.topic = (String) stormConf.get(TOPIC); + this.collector = collector; + } + + @Override + public void execute(Tuple input) { + K key = null; + if (input.contains(BOLT_KEY)) { + key = (K) input.getValueByField(BOLT_KEY); + } + V message = (V) input.getValueByField(BOLT_MESSAGE); + try { + producer.send(new KeyedMessage(topic, key, message)); + } catch (Exception ex) { + LOG.error("Could not send message with key '" + key + "' and value '" + message + "'", ex); + } finally { + collector.ack(input); + } + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + + } +} diff --git a/src/test/storm/kafka/bolt/KafkaBoltTest.java b/src/test/storm/kafka/bolt/KafkaBoltTest.java new file mode 100644 index 0000000000..129b0f6b68 --- /dev/null +++ b/src/test/storm/kafka/bolt/KafkaBoltTest.java @@ -0,0 +1,170 @@ +package storm.kafka.bolt; + +import backtype.storm.Config; +import backtype.storm.task.GeneralTopologyContext; +import backtype.storm.task.IOutputCollector; +import backtype.storm.task.OutputCollector; +import backtype.storm.topology.TopologyBuilder; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import backtype.storm.tuple.TupleImpl; +import backtype.storm.tuple.Values; +import backtype.storm.utils.Utils; +import kafka.api.OffsetRequest; +import kafka.javaapi.consumer.SimpleConsumer; +import kafka.javaapi.message.ByteBufferMessageSet; +import kafka.message.Message; +import kafka.message.MessageAndOffset; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import storm.kafka.*; +import storm.kafka.trident.GlobalPartitionInformation; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.verify; + +public class KafkaBoltTest { + + private static final String TEST_TOPIC = "test-topic"; + private KafkaTestBroker broker; + private KafkaBolt bolt; + private Config config = new Config(); + private KafkaConfig kafkaConfig; + private SimpleConsumer simpleConsumer; + + @Mock + private IOutputCollector collector; + + @Before + public void initMocks() { + MockitoAnnotations.initMocks(this); + broker = new KafkaTestBroker(); + setupKafkaConsumer(); + config.put(KafkaBolt.TOPIC, TEST_TOPIC); + bolt = generateStringSerializerBolt(); + } + + @After + public void shutdown() { + broker.shutdown(); + } + + + private void setupKafkaConsumer() { + GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(); + globalPartitionInformation.addPartition(0, Broker.fromString(broker.getBrokerConnectionString())); + BrokerHosts brokerHosts = new StaticHosts(globalPartitionInformation); + kafkaConfig = new KafkaConfig(brokerHosts, TEST_TOPIC); + simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient"); + } + + @Test + public void executeWithKey() throws Exception { + String message = "value-123"; + String key = "key-123"; + Tuple tuple = generateTestTuple(key, message); + bolt.execute(tuple); + verify(collector).ack(tuple); + verifyMessage(key, message); + } + + @Test + public void executeWithByteArrayKeyAndMessage() { + bolt = generateDefaultSerializerBolt(); + String keyString = "test-key"; + String messageString = "test-message"; + byte[] key = keyString.getBytes(); + byte[] message = messageString.getBytes(); + Tuple tuple = generateTestTuple(key, message); + bolt.execute(tuple); + verify(collector).ack(tuple); + verifyMessage(keyString, messageString); + } + + private KafkaBolt generateStringSerializerBolt() { + KafkaBolt bolt = new KafkaBolt(); + Properties props = new Properties(); + props.put("metadata.broker.list", broker.getBrokerConnectionString()); + props.put("request.required.acks", "1"); + props.put("serializer.class", "kafka.serializer.StringEncoder"); + config.put(KafkaBolt.KAFKA_BROKER_PROPERTIES, props); + bolt.prepare(config, null, new OutputCollector(collector)); + return bolt; + } + + private KafkaBolt generateDefaultSerializerBolt() { + KafkaBolt bolt = new KafkaBolt(); + Properties props = new Properties(); + props.put("metadata.broker.list", broker.getBrokerConnectionString()); + props.put("request.required.acks", "1"); + config.put(KafkaBolt.KAFKA_BROKER_PROPERTIES, props); + bolt.prepare(config, null, new OutputCollector(collector)); + return bolt; + } + + @Test + public void executeWithoutKey() throws Exception { + String message = "value-234"; + Tuple tuple = generateTestTuple(message); + bolt.execute(tuple); + verify(collector).ack(tuple); + verifyMessage(null, message); + } + + + @Test + public void executeWithBrokerDown() throws Exception { + broker.shutdown(); + String message = "value-234"; + Tuple tuple = generateTestTuple(message); + bolt.execute(tuple); + verify(collector).ack(tuple); + } + + + private boolean verifyMessage(String key, String message) { + long lastMessageOffset = KafkaUtils.getOffset(simpleConsumer, kafkaConfig.topic, 0, OffsetRequest.LatestTime()) - 1; + ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(kafkaConfig, simpleConsumer, + new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), lastMessageOffset); + MessageAndOffset messageAndOffset = messageAndOffsets.iterator().next(); + Message kafkaMessage = messageAndOffset.message(); + ByteBuffer messageKeyBuffer = kafkaMessage.key(); + String keyString = null; + String messageString = new String(Utils.toByteArray(kafkaMessage.payload())); + if (messageKeyBuffer != null) { + keyString = new String(Utils.toByteArray(messageKeyBuffer)); + } + assertEquals(key, keyString); + assertEquals(message, messageString); + return true; + } + + private Tuple generateTestTuple(Object key, Object message) { + TopologyBuilder builder = new TopologyBuilder(); + GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap(), new HashMap(), new HashMap(), "") { + @Override + public Fields getComponentOutputFields(String componentId, String streamId) { + return new Fields("key", "message"); + } + }; + return new TupleImpl(topologyContext, new Values(key, message), 1, ""); + } + + private Tuple generateTestTuple(Object message) { + TopologyBuilder builder = new TopologyBuilder(); + GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap(), new HashMap(), new HashMap(), "") { + @Override + public Fields getComponentOutputFields(String componentId, String streamId) { + return new Fields("message"); + } + }; + return new TupleImpl(topologyContext, new Values(message), 1, ""); + } +} From 6c82685c274c57576d45ab4d4b05ce03a3d2223b Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Fri, 28 Feb 2014 17:15:37 +0000 Subject: [PATCH 284/349] Work around ZOOKEEPER-1476 by prefering IPv4 by default. --- pom.xml | 1 + storm-core/pom.xml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/pom.xml b/pom.xml index c66ee22ca4..eee717e5e6 100644 --- a/pom.xml +++ b/pom.xml @@ -146,6 +146,7 @@ UTF-8 + -Djava.net.preferIPv4Stack=true 1.4.0 diff --git a/storm-core/pom.xml b/storm-core/pom.xml index 923c68d6b3..3506f15a75 100644 --- a/storm-core/pom.xml +++ b/storm-core/pom.xml @@ -205,6 +205,9 @@ test-with-junit + + ${test.extra.args} + From 70174deaa758fc1e3f3a881445e8750ae7218b3d Mon Sep 17 00:00:00 2001 From: supercargo Date: Fri, 28 Feb 2014 12:35:44 -0500 Subject: [PATCH 285/349] Moving chunk size out of configuration into private constant --- conf/defaults.yaml | 1 - storm-core/src/jvm/backtype/storm/Config.java | 6 ------ storm-core/src/jvm/backtype/storm/StormSubmitter.java | 6 +++--- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/conf/defaults.yaml b/conf/defaults.yaml index d0b87a053e..2dbba24c9f 100644 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@ -35,7 +35,6 @@ storm.zookeeper.retry.intervalceiling.millis: 30000 storm.cluster.mode: "distributed" # can be distributed or local storm.local.mode.zmq: false storm.thrift.transport: "backtype.storm.security.auth.SimpleTransportPlugin" -storm.thrift.chunk.size: 307200 storm.messaging.transport: "backtype.storm.messaging.netty.Context" ### nimbus.* configs are for the master diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java index 1a308c283f..281ae525df 100644 --- a/storm-core/src/jvm/backtype/storm/Config.java +++ b/storm-core/src/jvm/backtype/storm/Config.java @@ -136,12 +136,6 @@ public class Config extends HashMap { public static final String STORM_THRIFT_TRANSPORT_PLUGIN = "storm.thrift.transport"; public static final Object STORM_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class; - /** - * The chunk size in bytes to use when sending binary blobs over the Thrift protocol - */ - public static final String STORM_THRIFT_CHUNK_SIZE = "storm.thrift.chunk.size"; - public static final Object STORM_THRIFT_CHUNK_SIZE_SCHEMA = Number.class; - /** * The serializer class for ListDelegate (tuple payload). * The default serializer will be ListDelegateSerializer diff --git a/storm-core/src/jvm/backtype/storm/StormSubmitter.java b/storm-core/src/jvm/backtype/storm/StormSubmitter.java index b5081e8d44..26a0a78e3f 100644 --- a/storm-core/src/jvm/backtype/storm/StormSubmitter.java +++ b/storm-core/src/jvm/backtype/storm/StormSubmitter.java @@ -37,6 +37,8 @@ public class StormSubmitter { public static Logger LOG = LoggerFactory.getLogger(StormSubmitter.class); + private static final int THRIFT_CHUNK_SIZE_BYTES = 307200; + private static Nimbus.Iface localNimbus = null; public static void setLocalNimbus(Nimbus.Iface localNimbusHandler) { @@ -148,13 +150,11 @@ public static String submitJar(Map conf, String localJar) { throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload."); } NimbusClient client = NimbusClient.getConfiguredClient(conf); - - int thriftChunkSize = ((Number) conf.get(Config.STORM_THRIFT_CHUNK_SIZE)).intValue(); try { String uploadLocation = client.getClient().beginFileUpload(); LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation); - BufferFileInputStream is = new BufferFileInputStream(localJar, thriftChunkSize); + BufferFileInputStream is = new BufferFileInputStream(localJar, THRIFT_CHUNK_SIZE_BYTES); while(true) { byte[] toSubmit = is.read(); if(toSubmit.length==0) break; From bb8562f4f19dd04cbc941a4c32ed4cdb392f17a0 Mon Sep 17 00:00:00 2001 From: supercargo Date: Fri, 28 Feb 2014 12:37:06 -0500 Subject: [PATCH 286/349] whitespace cleanup --- storm-core/src/jvm/backtype/storm/StormSubmitter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/storm-core/src/jvm/backtype/storm/StormSubmitter.java b/storm-core/src/jvm/backtype/storm/StormSubmitter.java index 26a0a78e3f..53c8643155 100644 --- a/storm-core/src/jvm/backtype/storm/StormSubmitter.java +++ b/storm-core/src/jvm/backtype/storm/StormSubmitter.java @@ -150,7 +150,6 @@ public static String submitJar(Map conf, String localJar) { throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload."); } NimbusClient client = NimbusClient.getConfiguredClient(conf); - try { String uploadLocation = client.getClient().beginFileUpload(); LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation); From d9d637eab5f3761a4768fc960b2574d45962b466 Mon Sep 17 00:00:00 2001 From: Drew Date: Mon, 3 Mar 2014 12:08:43 -0800 Subject: [PATCH 287/349] Fixing an integer overflow in getSleepTimeMs(). This should fix the issue of negative values passed to sleep, causing IllegalArgumentExceptions. --- .../src/jvm/backtype/storm/messaging/netty/Client.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java b/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java index d765e7105f..0e62302135 100644 --- a/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java +++ b/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java @@ -40,8 +40,8 @@ class Client implements IConnection { private static final Logger LOG = LoggerFactory.getLogger(Client.class); private final int max_retries; - private final int base_sleep_ms; - private final int max_sleep_ms; + private final long base_sleep_ms; + private final long max_sleep_ms; private LinkedBlockingQueue message_queue; //entry should either be TaskMessage or ControlMessage private AtomicReference channelRef; private final ClientBootstrap bootstrap; @@ -107,10 +107,10 @@ void reconnect() { /** * # of milliseconds to wait per exponential back-off policy */ - private int getSleepTimeMs() + private long getSleepTimeMs() { int backoff = 1 << retries.get(); - int sleepMs = base_sleep_ms * Math.max(1, random.nextInt(backoff)); + long sleepMs = base_sleep_ms * Math.max(1, random.nextInt(backoff)); if ( sleepMs > max_sleep_ms ) sleepMs = max_sleep_ms; return sleepMs; From 286cae3815ae6888e463e75442f501d6e25f6dbb Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 5 Mar 2014 14:42:12 -0500 Subject: [PATCH 288/349] update CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6387df9f9..9c51b47e95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 0.9.2-incubating (unreleased) +* STORM-187: Fix Netty error "java.lang.IllegalArgumentException: timeout value is negative" + ## 0.9.1-incubating * Fix to prevent Nimbus from hanging if random data is sent to nimbus thrift port * Improved support for running on Windows platforms From 3aa6d7b95ef3cacccfacc02f0f35d1074780e2f1 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 5 Mar 2014 14:52:11 -0500 Subject: [PATCH 289/349] update CHANGELOG --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c51b47e95..554a05afc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## 0.9.2-incubating (unreleased) -* STORM-187: Fix Netty error "java.lang.IllegalArgumentException: timeout value is negative" - + * STORM-187: Fix Netty error "java.lang.IllegalArgumentException: timeout value is negative" + * STORM-186: fix float secs to millis long convertion ## 0.9.1-incubating * Fix to prevent Nimbus from hanging if random data is sent to nimbus thrift port * Improved support for running on Windows platforms From 8a2760d71d050093086f4ed5de28857d62ea263e Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 5 Mar 2014 16:48:01 -0500 Subject: [PATCH 290/349] add examples/storm-starter to build --- examples/storm-starter/pom.xml | 161 +++++++++++++++++++++++++++++++++ pom.xml | 9 +- 2 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 examples/storm-starter/pom.xml diff --git a/examples/storm-starter/pom.xml b/examples/storm-starter/pom.xml new file mode 100644 index 0000000000..3030a8fd79 --- /dev/null +++ b/examples/storm-starter/pom.xml @@ -0,0 +1,161 @@ + + + + 4.0.0 + + storm + org.apache.storm + 0.9.2-incubating-SNAPSHOT + ../../pom.xml + + + org.apache.storm + storm-starter + jar + + storm-starter + + + + + junit + junit + test + + + org.testng + testng + 6.8.5 + test + + + org.mockito + mockito-all + test + + + org.easytesting + fest-assert-core + 2.0M8 + test + + + org.jmock + jmock + 2.6.0 + test + + + org.apache.storm + storm-core + ${project.version} + + provided + + + commons-collections + commons-collections + 3.2.1 + + + com.google.guava + guava + + + + + src/jvm + test/jvm + + + ${basedir}/multilang + + + + + + + maven-assembly-plugin + + + jar-with-dependencies + + + + + + + + + + make-assembly + package + + single + + + + + + + com.theoryinpractise + clojure-maven-plugin + true + + + src/clj + + + + + compile + compile + + compile + + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + + + exec + + + + + java + true + false + compile + ${storm.topology} + + + + + diff --git a/pom.xml b/pom.xml index 7a3a840e40..24b3468e7e 100644 --- a/pom.xml +++ b/pom.xml @@ -130,6 +130,7 @@ storm-buildtools/maven-shade-clojure-transformer storm-core + examples/storm-starter @@ -416,7 +417,13 @@ 0.7.0 compile - + + + junit + junit + 3.8.1 + test + From a99d9c11be005ade7c308bebdda71c7fb0111acc Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 5 Mar 2014 17:10:35 -0500 Subject: [PATCH 291/349] add Apache license headers --- examples/storm-starter/.gitignore | 8 - examples/storm-starter/LICENSE | 14 -- examples/storm-starter/m2-pom.xml | 175 ------------------ .../multilang/resources/splitsentence.py | 15 ++ .../multilang/resources/splitsentence.rb | 15 ++ .../multilang/resources/storm.py | 15 ++ .../multilang/resources/storm.rb | 15 ++ examples/storm-starter/project.clj | 25 --- .../src/clj/storm/starter/clj/word_count.clj | 15 ++ .../jvm/storm/starter/BasicDRPCTopology.java | 17 ++ .../storm/starter/ExclamationTopology.java | 17 ++ .../src/jvm/storm/starter/ManualDRPC.java | 17 ++ .../jvm/storm/starter/PrintSampleStream.java | 17 ++ .../src/jvm/storm/starter/ReachTopology.java | 17 ++ .../jvm/storm/starter/RollingTopWords.java | 17 ++ .../jvm/storm/starter/SingleJoinExample.java | 17 ++ .../starter/TransactionalGlobalCount.java | 17 ++ .../jvm/storm/starter/TransactionalWords.java | 17 ++ .../jvm/storm/starter/WordCountTopology.java | 17 ++ .../starter/bolt/AbstractRankerBolt.java | 17 ++ .../bolt/IntermediateRankingsBolt.java | 17 ++ .../jvm/storm/starter/bolt/PrinterBolt.java | 17 ++ .../storm/starter/bolt/RollingCountBolt.java | 17 ++ .../storm/starter/bolt/SingleJoinBolt.java | 17 ++ .../storm/starter/bolt/TotalRankingsBolt.java | 17 ++ .../starter/spout/RandomSentenceSpout.java | 17 ++ .../starter/spout/TwitterSampleSpout.java | 17 ++ .../tools/NthLastModifiedTimeTracker.java | 17 ++ .../src/jvm/storm/starter/tools/Rankable.java | 17 ++ .../tools/RankableObjectWithFields.java | 17 ++ .../src/jvm/storm/starter/tools/Rankings.java | 17 ++ .../starter/tools/SlidingWindowCounter.java | 17 ++ .../storm/starter/tools/SlotBasedCounter.java | 17 ++ .../storm/starter/trident/TridentReach.java | 17 ++ .../starter/trident/TridentWordCount.java | 17 ++ .../jvm/storm/starter/util/StormRunner.java | 17 ++ .../jvm/storm/starter/util/TupleHelpers.java | 17 ++ examples/storm-starter/storm-starter.iml | 82 -------- .../bolt/IntermediateRankingsBoltTest.java | 17 ++ .../starter/bolt/RollingCountBoltTest.java | 17 ++ .../starter/bolt/TotalRankingsBoltTest.java | 17 ++ .../storm/starter/tools/MockTupleHelpers.java | 17 ++ .../tools/NthLastModifiedTimeTrackerTest.java | 17 ++ .../tools/RankableObjectWithFieldsTest.java | 17 ++ .../jvm/storm/starter/tools/RankingsTest.java | 17 ++ .../tools/SlidingWindowCounterTest.java | 17 ++ .../starter/tools/SlotBasedCounterTest.java | 17 ++ 47 files changed, 704 insertions(+), 304 deletions(-) delete mode 100644 examples/storm-starter/.gitignore delete mode 100644 examples/storm-starter/LICENSE delete mode 100644 examples/storm-starter/m2-pom.xml delete mode 100644 examples/storm-starter/project.clj delete mode 100644 examples/storm-starter/storm-starter.iml diff --git a/examples/storm-starter/.gitignore b/examples/storm-starter/.gitignore deleted file mode 100644 index c4b88f3834..0000000000 --- a/examples/storm-starter/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -classes/ -lib/ -*.pyc -.classpath -.project -*.jar -.lein-deps-sum - diff --git a/examples/storm-starter/LICENSE b/examples/storm-starter/LICENSE deleted file mode 100644 index 19560d1c38..0000000000 --- a/examples/storm-starter/LICENSE +++ /dev/null @@ -1,14 +0,0 @@ -Copyright 2011 Nathan Marz - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/examples/storm-starter/m2-pom.xml b/examples/storm-starter/m2-pom.xml deleted file mode 100644 index 77011a7e41..0000000000 --- a/examples/storm-starter/m2-pom.xml +++ /dev/null @@ -1,175 +0,0 @@ - - 4.0.0 - - storm.starter - storm-starter - 0.0.1-SNAPSHOT - jar - - storm-starter - https://github.com/nathanmarz/storm-starter - - - UTF-8 - - - - - github-releases - http://oss.sonatype.org/content/repositories/github-releases/ - - - clojars.org - http://clojars.org/repo - - - - - - junit - junit - 3.8.1 - test - - - org.testng - testng - 6.8.5 - test - - - org.mockito - mockito-all - 1.9.0 - test - - - org.easytesting - fest-assert-core - 2.0M8 - test - - - org.jmock - jmock - 2.6.0 - test - - - storm - storm - 0.9.0.1 - - provided - - - commons-collections - commons-collections - 3.2.1 - - - com.google.guava - guava - 15.0 - - - - - src/jvm - test/jvm - - - ${basedir}/multilang - - - - - - - maven-assembly-plugin - - - jar-with-dependencies - - - - - - - - - - make-assembly - package - - single - - - - - - - com.theoryinpractise - clojure-maven-plugin - 1.3.12 - true - - - src/clj - - - - - compile - compile - - compile - - - - test - test - - test - - - - - - - org.codehaus.mojo - exec-maven-plugin - 1.2.1 - - - - exec - - - - - java - true - false - compile - ${storm.topology} - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 1.6 - 1.6 - - - - - - diff --git a/examples/storm-starter/multilang/resources/splitsentence.py b/examples/storm-starter/multilang/resources/splitsentence.py index 24a9c9b82f..300105f2c3 100644 --- a/examples/storm-starter/multilang/resources/splitsentence.py +++ b/examples/storm-starter/multilang/resources/splitsentence.py @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import storm class SplitSentenceBolt(storm.BasicBolt): diff --git a/examples/storm-starter/multilang/resources/splitsentence.rb b/examples/storm-starter/multilang/resources/splitsentence.rb index 6b719ed099..6d1d2463df 100644 --- a/examples/storm-starter/multilang/resources/splitsentence.rb +++ b/examples/storm-starter/multilang/resources/splitsentence.rb @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. require "./storm" class SplitSentenceBolt < Storm::Bolt diff --git a/examples/storm-starter/multilang/resources/storm.py b/examples/storm-starter/multilang/resources/storm.py index c06b255733..4bc9e82486 100644 --- a/examples/storm-starter/multilang/resources/storm.py +++ b/examples/storm-starter/multilang/resources/storm.py @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import sys import os import traceback diff --git a/examples/storm-starter/multilang/resources/storm.rb b/examples/storm-starter/multilang/resources/storm.rb index 017fc25601..985b4123ca 100644 --- a/examples/storm-starter/multilang/resources/storm.rb +++ b/examples/storm-starter/multilang/resources/storm.rb @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. require "rubygems" require "json" diff --git a/examples/storm-starter/project.clj b/examples/storm-starter/project.clj deleted file mode 100644 index ae6c3778d1..0000000000 --- a/examples/storm-starter/project.clj +++ /dev/null @@ -1,25 +0,0 @@ -(defproject storm-starter "0.0.1-SNAPSHOT" - :source-paths ["src/clj"] - :java-source-paths ["src/jvm"] - :test-paths ["test/jvm"] - :resource-paths ["multilang"] - :aot :all - :repositories { -;; "twitter4j" "http://twitter4j.org/maven2" - } - - :dependencies [ -;; [org.twitter4j/twitter4j-core "2.2.6-SNAPSHOT"] -;; [org.twitter4j/twitter4j-stream "2.2.6-SNAPSHOT"] - [commons-collections/commons-collections "3.2.1"] - ] - - :profiles {:dev - {:dependencies [[storm "0.9.0.1"] - [org.clojure/clojure "1.4.0"] - [org.testng/testng "6.8.5"] - [org.easytesting/fest-assert-core "2.0M8"] - [org.mockito/mockito-all "1.9.0"] - [org.jmock/jmock "2.6.0"]]}} - :min-lein-version "2.0.0" - ) diff --git a/examples/storm-starter/src/clj/storm/starter/clj/word_count.clj b/examples/storm-starter/src/clj/storm/starter/clj/word_count.clj index ce2725d88a..3b54ac81e7 100644 --- a/examples/storm-starter/src/clj/storm/starter/clj/word_count.clj +++ b/examples/storm-starter/src/clj/storm/starter/clj/word_count.clj @@ -1,3 +1,18 @@ +;; Licensed to the Apache Software Foundation (ASF) under one +;; or more contributor license agreements. See the NOTICE file +;; distributed with this work for additional information +;; regarding copyright ownership. The ASF licenses this file +;; to you under the Apache License, Version 2.0 (the +;; "License"); you may not use this file except in compliance +;; with the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. (ns storm.starter.clj.word-count (:import [backtype.storm StormSubmitter LocalCluster]) (:use [backtype.storm clojure config]) diff --git a/examples/storm-starter/src/jvm/storm/starter/BasicDRPCTopology.java b/examples/storm-starter/src/jvm/storm/starter/BasicDRPCTopology.java index 7e7ef93a3b..b0493a784e 100644 --- a/examples/storm-starter/src/jvm/storm/starter/BasicDRPCTopology.java +++ b/examples/storm-starter/src/jvm/storm/starter/BasicDRPCTopology.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/ExclamationTopology.java b/examples/storm-starter/src/jvm/storm/starter/ExclamationTopology.java index fed8b1e780..a0fb757936 100644 --- a/examples/storm-starter/src/jvm/storm/starter/ExclamationTopology.java +++ b/examples/storm-starter/src/jvm/storm/starter/ExclamationTopology.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/ManualDRPC.java b/examples/storm-starter/src/jvm/storm/starter/ManualDRPC.java index ade4ab1908..fe0bae279d 100644 --- a/examples/storm-starter/src/jvm/storm/starter/ManualDRPC.java +++ b/examples/storm-starter/src/jvm/storm/starter/ManualDRPC.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java b/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java index e880e16229..8bead91ac8 100644 --- a/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java +++ b/examples/storm-starter/src/jvm/storm/starter/PrintSampleStream.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /* // to use this example, uncomment the twitter4j dependency information in the project.clj, // uncomment storm.starter.spout.TwitterSampleSpout, and uncomment this class diff --git a/examples/storm-starter/src/jvm/storm/starter/ReachTopology.java b/examples/storm-starter/src/jvm/storm/starter/ReachTopology.java index a63db3e3fa..de99a91f5d 100644 --- a/examples/storm-starter/src/jvm/storm/starter/ReachTopology.java +++ b/examples/storm-starter/src/jvm/storm/starter/ReachTopology.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/RollingTopWords.java b/examples/storm-starter/src/jvm/storm/starter/RollingTopWords.java index 9d0b3c822d..2630557db8 100644 --- a/examples/storm-starter/src/jvm/storm/starter/RollingTopWords.java +++ b/examples/storm-starter/src/jvm/storm/starter/RollingTopWords.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/SingleJoinExample.java b/examples/storm-starter/src/jvm/storm/starter/SingleJoinExample.java index d323809c1b..cb1d98c3a7 100644 --- a/examples/storm-starter/src/jvm/storm/starter/SingleJoinExample.java +++ b/examples/storm-starter/src/jvm/storm/starter/SingleJoinExample.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/TransactionalGlobalCount.java b/examples/storm-starter/src/jvm/storm/starter/TransactionalGlobalCount.java index 91b16aab7b..d8ff78dff2 100644 --- a/examples/storm-starter/src/jvm/storm/starter/TransactionalGlobalCount.java +++ b/examples/storm-starter/src/jvm/storm/starter/TransactionalGlobalCount.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/TransactionalWords.java b/examples/storm-starter/src/jvm/storm/starter/TransactionalWords.java index 4ee7b1271d..4d5ba1b179 100644 --- a/examples/storm-starter/src/jvm/storm/starter/TransactionalWords.java +++ b/examples/storm-starter/src/jvm/storm/starter/TransactionalWords.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/WordCountTopology.java b/examples/storm-starter/src/jvm/storm/starter/WordCountTopology.java index f26278cfaa..b8dece0120 100644 --- a/examples/storm-starter/src/jvm/storm/starter/WordCountTopology.java +++ b/examples/storm-starter/src/jvm/storm/starter/WordCountTopology.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/bolt/AbstractRankerBolt.java b/examples/storm-starter/src/jvm/storm/starter/bolt/AbstractRankerBolt.java index 07ac843355..cc5c0e7723 100644 --- a/examples/storm-starter/src/jvm/storm/starter/bolt/AbstractRankerBolt.java +++ b/examples/storm-starter/src/jvm/storm/starter/bolt/AbstractRankerBolt.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.bolt; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/bolt/IntermediateRankingsBolt.java b/examples/storm-starter/src/jvm/storm/starter/bolt/IntermediateRankingsBolt.java index 6a57d47028..d1805ff375 100644 --- a/examples/storm-starter/src/jvm/storm/starter/bolt/IntermediateRankingsBolt.java +++ b/examples/storm-starter/src/jvm/storm/starter/bolt/IntermediateRankingsBolt.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.bolt; import backtype.storm.tuple.Tuple; diff --git a/examples/storm-starter/src/jvm/storm/starter/bolt/PrinterBolt.java b/examples/storm-starter/src/jvm/storm/starter/bolt/PrinterBolt.java index 8b2e62bbf3..58fc8caf1a 100644 --- a/examples/storm-starter/src/jvm/storm/starter/bolt/PrinterBolt.java +++ b/examples/storm-starter/src/jvm/storm/starter/bolt/PrinterBolt.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.bolt; import backtype.storm.topology.BasicOutputCollector; diff --git a/examples/storm-starter/src/jvm/storm/starter/bolt/RollingCountBolt.java b/examples/storm-starter/src/jvm/storm/starter/bolt/RollingCountBolt.java index d5477496f3..f83906cea9 100644 --- a/examples/storm-starter/src/jvm/storm/starter/bolt/RollingCountBolt.java +++ b/examples/storm-starter/src/jvm/storm/starter/bolt/RollingCountBolt.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.bolt; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/bolt/SingleJoinBolt.java b/examples/storm-starter/src/jvm/storm/starter/bolt/SingleJoinBolt.java index e12f005a4b..85a7a267eb 100644 --- a/examples/storm-starter/src/jvm/storm/starter/bolt/SingleJoinBolt.java +++ b/examples/storm-starter/src/jvm/storm/starter/bolt/SingleJoinBolt.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.bolt; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/bolt/TotalRankingsBolt.java b/examples/storm-starter/src/jvm/storm/starter/bolt/TotalRankingsBolt.java index 217443e4c2..0e1bb05a77 100644 --- a/examples/storm-starter/src/jvm/storm/starter/bolt/TotalRankingsBolt.java +++ b/examples/storm-starter/src/jvm/storm/starter/bolt/TotalRankingsBolt.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.bolt; import backtype.storm.tuple.Tuple; diff --git a/examples/storm-starter/src/jvm/storm/starter/spout/RandomSentenceSpout.java b/examples/storm-starter/src/jvm/storm/starter/spout/RandomSentenceSpout.java index e808f8361c..813b10cf38 100644 --- a/examples/storm-starter/src/jvm/storm/starter/spout/RandomSentenceSpout.java +++ b/examples/storm-starter/src/jvm/storm/starter/spout/RandomSentenceSpout.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.spout; import backtype.storm.spout.SpoutOutputCollector; diff --git a/examples/storm-starter/src/jvm/storm/starter/spout/TwitterSampleSpout.java b/examples/storm-starter/src/jvm/storm/starter/spout/TwitterSampleSpout.java index 0a48cf2986..5cc12de163 100644 --- a/examples/storm-starter/src/jvm/storm/starter/spout/TwitterSampleSpout.java +++ b/examples/storm-starter/src/jvm/storm/starter/spout/TwitterSampleSpout.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /* package storm.starter.spout; diff --git a/examples/storm-starter/src/jvm/storm/starter/tools/NthLastModifiedTimeTracker.java b/examples/storm-starter/src/jvm/storm/starter/tools/NthLastModifiedTimeTracker.java index 92998a6d3e..08df8cfd03 100644 --- a/examples/storm-starter/src/jvm/storm/starter/tools/NthLastModifiedTimeTracker.java +++ b/examples/storm-starter/src/jvm/storm/starter/tools/NthLastModifiedTimeTracker.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.tools; import backtype.storm.utils.Time; diff --git a/examples/storm-starter/src/jvm/storm/starter/tools/Rankable.java b/examples/storm-starter/src/jvm/storm/starter/tools/Rankable.java index 254d920acb..85e3d1d0f8 100644 --- a/examples/storm-starter/src/jvm/storm/starter/tools/Rankable.java +++ b/examples/storm-starter/src/jvm/storm/starter/tools/Rankable.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.tools; public interface Rankable extends Comparable { diff --git a/examples/storm-starter/src/jvm/storm/starter/tools/RankableObjectWithFields.java b/examples/storm-starter/src/jvm/storm/starter/tools/RankableObjectWithFields.java index 3079434f6a..9a0ecae6ab 100644 --- a/examples/storm-starter/src/jvm/storm/starter/tools/RankableObjectWithFields.java +++ b/examples/storm-starter/src/jvm/storm/starter/tools/RankableObjectWithFields.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.tools; import backtype.storm.tuple.Tuple; diff --git a/examples/storm-starter/src/jvm/storm/starter/tools/Rankings.java b/examples/storm-starter/src/jvm/storm/starter/tools/Rankings.java index 5076bf65ac..551ebfbe3d 100644 --- a/examples/storm-starter/src/jvm/storm/starter/tools/Rankings.java +++ b/examples/storm-starter/src/jvm/storm/starter/tools/Rankings.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.tools; import com.google.common.collect.ImmutableList; diff --git a/examples/storm-starter/src/jvm/storm/starter/tools/SlidingWindowCounter.java b/examples/storm-starter/src/jvm/storm/starter/tools/SlidingWindowCounter.java index 38becdca8f..1199c40116 100644 --- a/examples/storm-starter/src/jvm/storm/starter/tools/SlidingWindowCounter.java +++ b/examples/storm-starter/src/jvm/storm/starter/tools/SlidingWindowCounter.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.tools; import java.io.Serializable; diff --git a/examples/storm-starter/src/jvm/storm/starter/tools/SlotBasedCounter.java b/examples/storm-starter/src/jvm/storm/starter/tools/SlotBasedCounter.java index f45ef48a1f..4b2d472dc9 100644 --- a/examples/storm-starter/src/jvm/storm/starter/tools/SlotBasedCounter.java +++ b/examples/storm-starter/src/jvm/storm/starter/tools/SlotBasedCounter.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.tools; import java.io.Serializable; diff --git a/examples/storm-starter/src/jvm/storm/starter/trident/TridentReach.java b/examples/storm-starter/src/jvm/storm/starter/trident/TridentReach.java index a940b40004..2d87c474eb 100644 --- a/examples/storm-starter/src/jvm/storm/starter/trident/TridentReach.java +++ b/examples/storm-starter/src/jvm/storm/starter/trident/TridentReach.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.trident; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/trident/TridentWordCount.java b/examples/storm-starter/src/jvm/storm/starter/trident/TridentWordCount.java index c56280c59b..439e00f488 100644 --- a/examples/storm-starter/src/jvm/storm/starter/trident/TridentWordCount.java +++ b/examples/storm-starter/src/jvm/storm/starter/trident/TridentWordCount.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.trident; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/util/StormRunner.java b/examples/storm-starter/src/jvm/storm/starter/util/StormRunner.java index 30c4f50d2c..f916ec61b8 100644 --- a/examples/storm-starter/src/jvm/storm/starter/util/StormRunner.java +++ b/examples/storm-starter/src/jvm/storm/starter/util/StormRunner.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.util; import backtype.storm.Config; diff --git a/examples/storm-starter/src/jvm/storm/starter/util/TupleHelpers.java b/examples/storm-starter/src/jvm/storm/starter/util/TupleHelpers.java index a46480e1bd..4ea669ed2e 100644 --- a/examples/storm-starter/src/jvm/storm/starter/util/TupleHelpers.java +++ b/examples/storm-starter/src/jvm/storm/starter/util/TupleHelpers.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.util; import backtype.storm.Constants; diff --git a/examples/storm-starter/storm-starter.iml b/examples/storm-starter/storm-starter.iml deleted file mode 100644 index a4e5116153..0000000000 --- a/examples/storm-starter/storm-starter.iml +++ /dev/null @@ -1,82 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/examples/storm-starter/test/jvm/storm/starter/bolt/IntermediateRankingsBoltTest.java b/examples/storm-starter/test/jvm/storm/starter/bolt/IntermediateRankingsBoltTest.java index cbbee3ce80..c296a899e4 100644 --- a/examples/storm-starter/test/jvm/storm/starter/bolt/IntermediateRankingsBoltTest.java +++ b/examples/storm-starter/test/jvm/storm/starter/bolt/IntermediateRankingsBoltTest.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.bolt; import backtype.storm.Config; diff --git a/examples/storm-starter/test/jvm/storm/starter/bolt/RollingCountBoltTest.java b/examples/storm-starter/test/jvm/storm/starter/bolt/RollingCountBoltTest.java index c9a516a385..bc31ba06d4 100644 --- a/examples/storm-starter/test/jvm/storm/starter/bolt/RollingCountBoltTest.java +++ b/examples/storm-starter/test/jvm/storm/starter/bolt/RollingCountBoltTest.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.bolt; import backtype.storm.Config; diff --git a/examples/storm-starter/test/jvm/storm/starter/bolt/TotalRankingsBoltTest.java b/examples/storm-starter/test/jvm/storm/starter/bolt/TotalRankingsBoltTest.java index a8bed45f8f..49e3d679f2 100644 --- a/examples/storm-starter/test/jvm/storm/starter/bolt/TotalRankingsBoltTest.java +++ b/examples/storm-starter/test/jvm/storm/starter/bolt/TotalRankingsBoltTest.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.bolt; import backtype.storm.Config; diff --git a/examples/storm-starter/test/jvm/storm/starter/tools/MockTupleHelpers.java b/examples/storm-starter/test/jvm/storm/starter/tools/MockTupleHelpers.java index fd7d921407..b253350ef4 100644 --- a/examples/storm-starter/test/jvm/storm/starter/tools/MockTupleHelpers.java +++ b/examples/storm-starter/test/jvm/storm/starter/tools/MockTupleHelpers.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.tools; import backtype.storm.Constants; diff --git a/examples/storm-starter/test/jvm/storm/starter/tools/NthLastModifiedTimeTrackerTest.java b/examples/storm-starter/test/jvm/storm/starter/tools/NthLastModifiedTimeTrackerTest.java index c79d3825ef..fe4d987d87 100644 --- a/examples/storm-starter/test/jvm/storm/starter/tools/NthLastModifiedTimeTrackerTest.java +++ b/examples/storm-starter/test/jvm/storm/starter/tools/NthLastModifiedTimeTrackerTest.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.tools; import backtype.storm.utils.Time; diff --git a/examples/storm-starter/test/jvm/storm/starter/tools/RankableObjectWithFieldsTest.java b/examples/storm-starter/test/jvm/storm/starter/tools/RankableObjectWithFieldsTest.java index 82a89e6dfc..e83f922094 100644 --- a/examples/storm-starter/test/jvm/storm/starter/tools/RankableObjectWithFieldsTest.java +++ b/examples/storm-starter/test/jvm/storm/starter/tools/RankableObjectWithFieldsTest.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.tools; import backtype.storm.tuple.Tuple; diff --git a/examples/storm-starter/test/jvm/storm/starter/tools/RankingsTest.java b/examples/storm-starter/test/jvm/storm/starter/tools/RankingsTest.java index e1f82903cf..cab02cbe11 100644 --- a/examples/storm-starter/test/jvm/storm/starter/tools/RankingsTest.java +++ b/examples/storm-starter/test/jvm/storm/starter/tools/RankingsTest.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.tools; import com.google.common.base.Throwables; diff --git a/examples/storm-starter/test/jvm/storm/starter/tools/SlidingWindowCounterTest.java b/examples/storm-starter/test/jvm/storm/starter/tools/SlidingWindowCounterTest.java index 1c883e573a..920bf017e8 100644 --- a/examples/storm-starter/test/jvm/storm/starter/tools/SlidingWindowCounterTest.java +++ b/examples/storm-starter/test/jvm/storm/starter/tools/SlidingWindowCounterTest.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.tools; import org.testng.annotations.DataProvider; diff --git a/examples/storm-starter/test/jvm/storm/starter/tools/SlotBasedCounterTest.java b/examples/storm-starter/test/jvm/storm/starter/tools/SlotBasedCounterTest.java index aeeec63407..3ad042beb8 100644 --- a/examples/storm-starter/test/jvm/storm/starter/tools/SlotBasedCounterTest.java +++ b/examples/storm-starter/test/jvm/storm/starter/tools/SlotBasedCounterTest.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.starter.tools; import org.testng.annotations.DataProvider; From 651df24374b0b5de45e0a65155fba02af95cf020 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 5 Mar 2014 17:21:11 -0500 Subject: [PATCH 292/349] update README to reflect incorporation into storm source tree --- examples/storm-starter/README.markdown | 40 +++++++------------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/examples/storm-starter/README.markdown b/examples/storm-starter/README.markdown index e271ac1b03..13dd9e4939 100644 --- a/examples/storm-starter/README.markdown +++ b/examples/storm-starter/README.markdown @@ -7,7 +7,6 @@ Learn to use Storm! Table of Contents * Getting started -* Using storm-starter with Leiningen * Using storm-starter with Maven * Using storm-starter with IntelliJ IDEA @@ -27,7 +26,7 @@ Next, make sure you have the storm-starter code available on your machine. Git/ following command to download the latest storm-starter code and change to the new directory that contains the downloaded code. - $ git clone git://github.com/nathanmarz/storm-starter.git && cd storm-starter + $ git clone git://github.com/apache/incubator-storm.git && cd storm/examples/storm-starter ## storm-starter overview @@ -45,7 +44,7 @@ After you have familiarized yourself with these topologies, take a look at the o for more advanced implementations. If you want to learn more about how Storm works, please head over to the -[Storm project page](http://github.com/nathanmarz/storm). +[Storm project page](http://storm.incubator.apache.org). @@ -58,48 +57,32 @@ The storm-starter build uses [Leiningen](http://leiningen.org/) 2.0. Install Le [leiningen installation instructions](https://github.com/technomancy/leiningen). -## Running topologies with Leiningen - -### To run a Java topology - - $ lein deps - $ lein compile - $ java -cp $(lein classpath) storm.starter.ExclamationTopology - - -### To run a Clojure topology: - - $ lein deps - $ lein compile - $ lein run -m storm.starter.clj.word-count - - # Using storm-starter with Maven ## Install Maven -[Maven](http://maven.apache.org/) is an alternative to Leiningen. Install Maven (preferably version 3.x) by following +Install [Maven](http://maven.apache.org/) (preferably version 3.x) by following the [Maven installation instructions](http://maven.apache.org/download.cgi). ## Running topologies with Maven -storm-starter contains [m2-pom.xml](m2-pom.xml) which can be used with Maven using the `-f` option. For example, to +storm-starter topologies can be run with the maven-exec-plugin. For example, to compile and run `WordCountTopology` in local mode, use the command: - $ mvn -f m2-pom.xml compile exec:java -Dstorm.topology=storm.starter.WordCountTopology + $ mvn compile exec:java -Dstorm.topology=storm.starter.WordCountTopology You can also run clojure topologies with Maven: - $ mvn -f m2-pom.xml compile exec:java -Dstorm.topology=storm.starter.clj.word_count + $ mvn compile exec:java -Dstorm.topology=storm.starter.clj.word_count ## Packaging storm-starter for use on a Storm cluster You can package a jar suitable for submitting to a Storm cluster with the command: - $ mvn -f m2-pom.xml package + $ mvn package This will package your code and all the non-Storm dependencies into a single "uberjar" at the path `target/storm-starter-{version}-jar-with-dependencies.jar`. @@ -110,7 +93,7 @@ This will package your code and all the non-Storm dependencies into a single "ub Use the following Maven command to run the unit tests that ship with storm-starter. Unfortunately `lein test` does not yet run the included unit tests. - $ mvn -f m2-pom.xml test + $ mvn test @@ -121,10 +104,9 @@ yet run the included unit tests. The following instructions will import storm-starter as a new project in IntelliJ IDEA. -* Copy `m2-pom.xml` to `pom.xml`. This is requried so that IDEA (or Eclipse) can properly detect the maven - configuration. -* Open _File > Import Project..._ and navigate to the top-level directory of your storm-starter clone (e.g. - `~/git/storm-starter`). + +* Open _File > Import Project..._ and navigate to the storm-starter directory of your storm clone (e.g. + `~/git/storm/examples/storm-starter`). * Select _Import project from external model_, select "Maven", and click _Next_. * In the following screen, enable the checkbox _Import Maven projects automatically_. Leave all other values at their defaults. Click _Next_. From 017d73f20d78f61761150950e0e1df6da640499a Mon Sep 17 00:00:00 2001 From: nathanmarz Date: Tue, 11 Mar 2014 00:02:17 +0000 Subject: [PATCH 293/349] remove nathan from notices --- NOTICE | 4 ---- storm-dist/binary/NOTICE | 7 +------ 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/NOTICE b/NOTICE index 5bdb68ccf9..35e9475a71 100644 --- a/NOTICE +++ b/NOTICE @@ -4,9 +4,5 @@ Copyright 2014 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). -This product includes software developed by Nathan Marz -Copyright 2011-2013 Nathan Marz - - This product includes software developed by Yahoo! Inc. (www.yahoo.com) Copyright © 2012-2013 Yahoo! Inc. All rights reserved. diff --git a/storm-dist/binary/NOTICE b/storm-dist/binary/NOTICE index 0eec4766f8..5fe5e4f299 100644 --- a/storm-dist/binary/NOTICE +++ b/storm-dist/binary/NOTICE @@ -4,11 +4,6 @@ Copyright 2013 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). - -This product includes software developed by Nathan Marz -Copyright 2011-2013 Nathan Marz - - This product includes software developed by Yahoo! Inc. (www.yahoo.com) Copyright © 2012-2013 Yahoo! Inc. All rights reserved. @@ -33,4 +28,4 @@ Copyright (C) 2009 Fang Yidong and Chris Nokleberg Alternative collection types provided by google-collections from http://code.google.com/p/google-collections/. -Copyright (C) 2007 Google Inc. \ No newline at end of file +Copyright (C) 2007 Google Inc. From b25eeedffb5e9553e8d052a5da47c212fcf53a89 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Tue, 11 Mar 2014 14:18:10 -0400 Subject: [PATCH 294/349] update commons-io dependency version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7a3a840e40..0fe9ad5796 100644 --- a/pom.xml +++ b/pom.xml @@ -151,7 +151,7 @@ 1.4.0 1.1.3 0.3.6 - 1.4 + 2.4 2.5 1.1 0.4.1 From bb13ccedc0ca7443b46c3698ed864bbb19f00220 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 12 Mar 2014 13:02:57 -0400 Subject: [PATCH 295/349] fix spacing issue --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 24b3468e7e..a42cd4b28e 100644 --- a/pom.xml +++ b/pom.xml @@ -130,7 +130,7 @@ storm-buildtools/maven-shade-clojure-transformer storm-core - examples/storm-starter + examples/storm-starter From 82a754a520f3633903b90af7a03d8ee313ec0c46 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 12 Mar 2014 13:52:18 -0400 Subject: [PATCH 296/349] add examples source code to binary distribution --- storm-dist/binary/src/main/assembly/binary.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/storm-dist/binary/src/main/assembly/binary.xml b/storm-dist/binary/src/main/assembly/binary.xml index 36a03190e7..66b6f498a8 100644 --- a/storm-dist/binary/src/main/assembly/binary.xml +++ b/storm-dist/binary/src/main/assembly/binary.xml @@ -52,11 +52,23 @@ + + ${project.basedir}/../../examples + examples + + **/target/** + + + + ${project.basedir}/../../examples/storm-starter/target/storm-starter-${project.version}-jar-with-dependencies.jar + /examples/storm-starter/ + storm-starter-topologies-${project.version}.jar + From 1e6e080118d4d0681eca90ecaad01e7327ade0fe Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 12 Mar 2014 15:18:14 -0400 Subject: [PATCH 297/349] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 554a05afc3..c0f8dd3d1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## 0.9.2-incubating (unreleased) * STORM-187: Fix Netty error "java.lang.IllegalArgumentException: timeout value is negative" * STORM-186: fix float secs to millis long convertion + * STORM-70: Upgrade to ZK-3.4.5 and curator-1.3.3 + * STORM-146: Unit test regression when storm is compiled with 3.4.5 zookeeper ## 0.9.1-incubating * Fix to prevent Nimbus from hanging if random data is sent to nimbus thrift port * Improved support for running on Windows platforms From a2be522877a5a86c0c4e0f3407b81e7f8e99c95d Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 12 Mar 2014 15:18:14 -0400 Subject: [PATCH 298/349] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 554a05afc3..c0f8dd3d1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## 0.9.2-incubating (unreleased) * STORM-187: Fix Netty error "java.lang.IllegalArgumentException: timeout value is negative" * STORM-186: fix float secs to millis long convertion + * STORM-70: Upgrade to ZK-3.4.5 and curator-1.3.3 + * STORM-146: Unit test regression when storm is compiled with 3.4.5 zookeeper ## 0.9.1-incubating * Fix to prevent Nimbus from hanging if random data is sent to nimbus thrift port * Improved support for running on Windows platforms From 69594018082333f6dc168a3e5e07ee2eb9a75c8f Mon Sep 17 00:00:00 2001 From: nathanmarz Date: Mon, 17 Mar 2014 14:07:45 -0700 Subject: [PATCH 299/349] adds capability to remove keys from memorymapstate (maintains proper opaque semantics) --- .../storm/trident/state/map/OpaqueMap.java | 6 +++- .../trident/state/map/RemovableMapState.java | 8 +++++ .../storm/trident/testing/MemoryMapState.java | 27 ++++++++++++++- .../test/clj/storm/trident/state_test.clj | 33 ++++++++++++++++++- 4 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 storm-core/src/jvm/storm/trident/state/map/RemovableMapState.java diff --git a/storm-core/src/jvm/storm/trident/state/map/OpaqueMap.java b/storm-core/src/jvm/storm/trident/state/map/OpaqueMap.java index cd6766d16c..12f3083649 100644 --- a/storm-core/src/jvm/storm/trident/state/map/OpaqueMap.java +++ b/storm-core/src/jvm/storm/trident/state/map/OpaqueMap.java @@ -43,7 +43,11 @@ public List multiGet(List> keys) { for(CachedBatchReadsMap.RetVal retval: curr) { OpaqueValue val = retval.val; if(val!=null) { - ret.add((T) val.get(_currTx)); + if(retval.cached) { + ret.add((T) val.getCurr()); + } else { + ret.add((T) val.get(_currTx)); + } } else { ret.add(null); } diff --git a/storm-core/src/jvm/storm/trident/state/map/RemovableMapState.java b/storm-core/src/jvm/storm/trident/state/map/RemovableMapState.java new file mode 100644 index 0000000000..cf34f05e88 --- /dev/null +++ b/storm-core/src/jvm/storm/trident/state/map/RemovableMapState.java @@ -0,0 +1,8 @@ +package storm.trident.state.map; + +import java.util.List; +import storm.trident.state.State; + +public interface RemovableMapState extends State { + void multiRemove(List> keys); +} diff --git a/storm-core/src/jvm/storm/trident/testing/MemoryMapState.java b/storm-core/src/jvm/storm/trident/testing/MemoryMapState.java index 5df99f781a..fd38900de1 100644 --- a/storm-core/src/jvm/storm/trident/testing/MemoryMapState.java +++ b/storm-core/src/jvm/storm/trident/testing/MemoryMapState.java @@ -30,10 +30,13 @@ import storm.trident.state.map.*; import storm.trident.state.snapshot.Snapshottable; -public class MemoryMapState implements Snapshottable, ITupleCollection, MapState { +public class MemoryMapState implements Snapshottable, ITupleCollection, MapState, RemovableMapState { MemoryMapStateBacking _backing; SnapshottableMap _delegate; + List> _removed = new ArrayList(); + Long _currTx = null; + public MemoryMapState(String id) { _backing = new MemoryMapStateBacking(id); @@ -54,6 +57,11 @@ public T get() { public void beginCommit(Long txid) { _delegate.beginCommit(txid); + if(txid==null || !txid.equals(_currTx)) { + _backing.multiRemove(_removed); + } + _removed = new ArrayList(); + _currTx = txid; } public void commit(Long txid) { @@ -76,6 +84,17 @@ public List multiGet(List> keys) { return _delegate.multiGet(keys); } + @Override + public void multiRemove(List> keys) { + List nulls = new ArrayList(); + for(int i=0; i, T>) _dbs.get(id); } + public void multiRemove(List> keys) { + for(List key: keys) { + db.remove(key); + } + } + @Override public List multiGet(List> keys) { List ret = new ArrayList(); diff --git a/storm-core/test/clj/storm/trident/state_test.clj b/storm-core/test/clj/storm/trident/state_test.clj index 6e091e34f5..63e38cab98 100644 --- a/storm-core/test/clj/storm/trident/state_test.clj +++ b/storm-core/test/clj/storm/trident/state_test.clj @@ -20,10 +20,16 @@ (:import [storm.trident.state OpaqueValue]) (:import [storm.trident.state CombinerValueUpdater]) (:import [storm.trident.state.map TransactionalMap OpaqueMap]) - (:import [storm.trident.testing MemoryBackingMap]) + (:import [storm.trident.testing MemoryBackingMap MemoryMapState]) (:use [storm.trident testing]) (:use [backtype.storm util])) +(defn single-remove [map key] + (-> map (.multiRemove [[key]]))) + +(defn single-put [map key val] + (-> map (.multiPut [[key]] [val]))) + (defn single-get [map key] (-> map (.multiGet [[key]]) first)) @@ -61,7 +67,9 @@ (is (= nil (single-get map "a"))) ;; tests that intra-batch caching works (is (= 1 (single-update map "a" 1))) + (is (= 1 (single-get map "a"))) (is (= 3 (single-update map "a" 2))) + (is (= 3 (single-get map "a"))) (.commit map 1) (.beginCommit map 1) (is (= nil (single-get map "a"))) @@ -94,3 +102,26 @@ (is (= 7 (single-update map "a" 1))) (.commit map 2) )) + + +(deftest test-memory-map-state-remove + (let [map (MemoryMapState. (uuid))] + (.beginCommit map 1) + (single-put map "a" 1) + (single-put map "b" 2) + (.commit map 1) + (.beginCommit map 2) + (single-remove map "a") + (is (nil? (single-get map "a"))) + (is (= 2 (single-get map "b"))) + (.commit map 2) + (.beginCommit map 2) + (is (= 1 (single-get map "a"))) + (is (= 2 (single-get map "b"))) + (single-remove map "a") + (.commit map 2) + (.beginCommit map 3) + (is (nil? (single-get map "a"))) + (is (= 2 (single-get map "b"))) + (.commit map 3) + )) From c62d2e115fd2fcbdb2f8030913ac8492ea59c302 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Tue, 18 Mar 2014 11:42:48 -0400 Subject: [PATCH 300/349] update storm to 0.9.1-incubating --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 910041ad90..a9ceef1a45 100644 --- a/pom.xml +++ b/pom.xml @@ -168,9 +168,9 @@ - storm - storm - 0.9.0 + org.apache.storm + storm-core + 0.9.1-incubating provided From 84f04c3fea11ccb6844bdef7baf8e6f654637981 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Tue, 18 Mar 2014 15:36:09 -0400 Subject: [PATCH 301/349] align with 0.9.2-incubating dependencies. --- pom.xml | 117 ++---------------- src/jvm/storm/kafka/DynamicBrokersReader.java | 18 ++- 2 files changed, 17 insertions(+), 118 deletions(-) diff --git a/pom.xml b/pom.xml index 4626fbd8f7..e6adc5d910 100644 --- a/pom.xml +++ b/pom.xml @@ -2,26 +2,12 @@ 4.0.0 - net.wurstmeister.storm - storm-kafka-0.8-plus + org.apache.storm + storm-kafka jar - 0.5.0-SNAPSHOT - storm-kafka-0.8-plus - Storm module for kafka > 0.8 - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - A business-friendly OSS license - - - https://github.com/wurstmeister/storm-kafka-0.8-plus - - scm:git:git://github.com/wurstmeister/storm-kafka-0.8-plus.git - scm:git:ssh://git@github.com/wurstmeister/storm-kafka-0.8-plus.git - https://github.com/wurstmeister/storm-kafka-0.8-plus - + 0.9.2-incubating-SNAPSHOT + storm-kafka + Storm Spouts for Apache Kafka 2.9.2 kafka_2.9.2 @@ -29,74 +15,11 @@ - - maven-jar-plugin - 2.4 - - ${envClassifier} - - + src/jvm src/test - - - resources - - - - - dev-resources - - - resources - - - target - target/classes - - - central - http://repo1.maven.org/maven2/ - - false - - - true - - - - clojars - https://clojars.org/repo/ - - true - - - true - - - - scala-tools - http://scala-tools.org/repo-releases - - true - - - true - - - - conjars - http://conjars.org/repo/ - - true - - - true - - - org.mockito @@ -118,7 +41,7 @@ com.netflix.curator curator-framework - 1.0.1 + 1.3.3 log4j @@ -133,7 +56,7 @@ com.netflix.curator curator-recipes - 1.0.1 + 1.3.3 log4j @@ -145,7 +68,7 @@ com.netflix.curator curator-test - 1.0.1 + 1.3.3 log4j @@ -176,30 +99,10 @@ org.apache.storm storm-core - 0.9.1-incubating - provided - - - org.slf4j - log4j-over-slf4j - 1.6.6 - provided - - - org.clojure - clojure - 1.4.0 + 0.9.2-incubating-SNAPSHOT provided - - - clojars - Clojars repository - https://clojars.org/repo - - - Scala-2.10 diff --git a/src/jvm/storm/kafka/DynamicBrokersReader.java b/src/jvm/storm/kafka/DynamicBrokersReader.java index 5b1d750bab..cd751fefb2 100644 --- a/src/jvm/storm/kafka/DynamicBrokersReader.java +++ b/src/jvm/storm/kafka/DynamicBrokersReader.java @@ -26,17 +26,13 @@ public class DynamicBrokersReader { public DynamicBrokersReader(Map conf, String zkStr, String zkPath, String topic) { _zkPath = zkPath; _topic = topic; - try { - _curator = CuratorFrameworkFactory.newClient( - zkStr, - Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), - 15000, - new RetryNTimes(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), - Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); - _curator.start(); - } catch (IOException ex) { - LOG.error("can't connect to zookeeper"); - } + _curator = CuratorFrameworkFactory.newClient( + zkStr, + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), + 15000, + new RetryNTimes(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); + _curator.start(); } /** From 20b3496c258e356664604912df8df3711914dd95 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Thu, 20 Mar 2014 17:23:47 -0400 Subject: [PATCH 302/349] update README with new committers --- README.markdown | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.markdown b/README.markdown index 525463922a..46ff724f72 100644 --- a/README.markdown +++ b/README.markdown @@ -71,6 +71,9 @@ under the License. * Flip Kromer ([@mrflip](https://github.com/mrflip)) * David Lao ([@davidlao2k](https://github.com/davidlao2k)) * P. Taylor Goetz ([@ptgoetz](https://github.com/ptgoetz)) +* Derek Dagit ([@d2r](https://github.com/d2r)) +* Robert Evans ([@revans2](https://github.com/revans2)) + ## Contributors @@ -105,9 +108,7 @@ under the License. * Bryan Peterson ([@Lazyshot](https://github.com/Lazyshot)) * Sam Ritchie ([@sritchie](https://github.com/sritchie)) * Stuart Anderson ([@emblem](https://github.com/emblem)) -* Robert Evans ([@revans2](https://github.com/revans2)) * Lorcan Coyle ([@lorcan](https://github.com/lorcan)) -* Derek Dagit ([@d2r](https://github.com/d2r)) * Andrew Olson ([@noslowerdna](https://github.com/noslowerdna)) * Gavin Li ([@lyogavin](https://github.com/lyogavin)) * Tudor Scurtu ([@tscurtu](https://github.com/tscurtu)) From d5dee0ef5f8ad403d1eff05f569c8e9b1e44508c Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Thu, 20 Mar 2014 17:36:18 -0400 Subject: [PATCH 303/349] STORM-259 --- storm-core/src/jvm/backtype/storm/messaging/netty/Client.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java b/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java index 0e62302135..f15cd1d412 100644 --- a/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java +++ b/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java @@ -109,6 +109,9 @@ void reconnect() { */ private long getSleepTimeMs() { + if (retries.get() > 30) { + return max_sleep_ms; + } int backoff = 1 << retries.get(); long sleepMs = base_sleep_ms * Math.max(1, random.nextInt(backoff)); if ( sleepMs > max_sleep_ms ) From 28e65a82bfcd60f38c93115974f6a8653756f9b5 Mon Sep 17 00:00:00 2001 From: Adrian Petrescu Date: Fri, 21 Mar 2014 17:17:40 -0400 Subject: [PATCH 304/349] Fix race condition in Time.java Some of my test runs were occasionally failing with a NullPointerException on `backtype.storm.utils.Time.java:64`. After a bit of investigation, it seems there's a race condition here; if we disable simulating mode while a thread is currently sleeping, then when it wakes up it won't re-check if it's still in "simulating" mode, it'll try to remove the sleep time, and get the NPE. Refs [STORM-260](https://issues.apache.org/jira/browse/STORM-260). --- .../src/jvm/backtype/storm/utils/Time.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/storm-core/src/jvm/backtype/storm/utils/Time.java b/storm-core/src/jvm/backtype/storm/utils/Time.java index 506b0746ef..50a79fd673 100644 --- a/storm-core/src/jvm/backtype/storm/utils/Time.java +++ b/storm-core/src/jvm/backtype/storm/utils/Time.java @@ -36,14 +36,18 @@ public class Time { private static AtomicLong simulatedCurrTimeMs; //should this be a thread local that's allowed to keep advancing? public static void startSimulating() { - simulating.set(true); - simulatedCurrTimeMs = new AtomicLong(0); - threadSleepTimes = new ConcurrentHashMap(); + synchronized(sleepTimesLock) { + simulating.set(true); + simulatedCurrTimeMs = new AtomicLong(0); + threadSleepTimes = new ConcurrentHashMap(); + } } public static void stopSimulating() { - simulating.set(false); - threadSleepTimes = null; + synchronized(sleepTimesLock) { + simulating.set(false); + threadSleepTimes = null; + } } public static boolean isSimulating() { @@ -61,7 +65,9 @@ public static void sleepUntil(long targetTimeMs) throws InterruptedException { } } finally { synchronized(sleepTimesLock) { - threadSleepTimes.remove(Thread.currentThread()); + if (simulating.get()) { + threadSleepTimes.remove(Thread.currentThread()); + } } } } else { From 813952a8b1345586e53da62d1d65546c15d9e120 Mon Sep 17 00:00:00 2001 From: Niels Basjes Date: Mon, 24 Mar 2014 20:06:51 +0100 Subject: [PATCH 305/349] Bolt with whitespace in name can be viewed in ui --- storm-core/src/clj/backtype/storm/ui/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj index 63a3053d52..27e3b15476 100644 --- a/storm-core/src/clj/backtype/storm/ui/core.clj +++ b/storm-core/src/clj/backtype/storm/ui/core.clj @@ -1024,7 +1024,7 @@ (catch Exception e (resp/redirect "/"))))) (GET "/topology/:id/component/:component" [:as {cookies :cookies} id component & m] (let [include-sys? (get-include-sys? cookies)] - (-> (component-page id component (:window m) include-sys?) + (-> (component-page id (java.net.URLDecoder/decode component) (:window m) include-sys?) (concat [(mk-system-toggle-button include-sys?)]) ui-template))) (POST "/topology/:id/activate" [id] From dbf8fafeceb6c6e77c91389e75424384d4909baa Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Mon, 24 Mar 2014 20:43:44 +0100 Subject: [PATCH 306/349] storm-starter: fix base directory path to point to `incubator-storm` --- examples/storm-starter/README.markdown | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/storm-starter/README.markdown b/examples/storm-starter/README.markdown index 13dd9e4939..32e21bc1fa 100644 --- a/examples/storm-starter/README.markdown +++ b/examples/storm-starter/README.markdown @@ -26,7 +26,7 @@ Next, make sure you have the storm-starter code available on your machine. Git/ following command to download the latest storm-starter code and change to the new directory that contains the downloaded code. - $ git clone git://github.com/apache/incubator-storm.git && cd storm/examples/storm-starter + $ git clone git://github.com/apache/incubator-storm.git && cd incubator-storm/examples/storm-starter ## storm-starter overview @@ -106,7 +106,7 @@ The following instructions will import storm-starter as a new project in Intelli * Open _File > Import Project..._ and navigate to the storm-starter directory of your storm clone (e.g. - `~/git/storm/examples/storm-starter`). + `~/git/incubator-storm/examples/storm-starter`). * Select _Import project from external model_, select "Maven", and click _Next_. * In the following screen, enable the checkbox _Import Maven projects automatically_. Leave all other values at their defaults. Click _Next_. From be8a22b0541e0fe7d5dcbc4c83b17d3b92328e99 Mon Sep 17 00:00:00 2001 From: "Michael G. Noll" Date: Mon, 24 Mar 2014 20:47:37 +0100 Subject: [PATCH 307/349] storm-starter: remove leiningen section because leiningen is not used anymore --- examples/storm-starter/README.markdown | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/examples/storm-starter/README.markdown b/examples/storm-starter/README.markdown index 13dd9e4939..f10f444c9a 100644 --- a/examples/storm-starter/README.markdown +++ b/examples/storm-starter/README.markdown @@ -47,16 +47,6 @@ If you want to learn more about how Storm works, please head over to the [Storm project page](http://storm.incubator.apache.org). - - -# Using storm-starter with Leiningen - -## Install Leiningen - -The storm-starter build uses [Leiningen](http://leiningen.org/) 2.0. Install Leiningen by following the -[leiningen installation instructions](https://github.com/technomancy/leiningen). - - # Using storm-starter with Maven From df687cb71ba174e408a6cf68011053ff18b16f71 Mon Sep 17 00:00:00 2001 From: Niels Basjes Date: Mon, 24 Mar 2014 22:06:32 +0100 Subject: [PATCH 308/349] Ensure that no evil chars can be put into the HTML via the the name of a component --- storm-core/src/clj/backtype/storm/ui/core.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj index 27e3b15476..3b975c5a62 100644 --- a/storm-core/src/clj/backtype/storm/ui/core.clj +++ b/storm-core/src/clj/backtype/storm/ui/core.clj @@ -485,7 +485,7 @@ :let [disp ((display-map k) k)]] [(link-to (if (= k window) {:class "red"} {}) (url-format "/topology/%s?window=%s" id k) - disp) + (escape-html disp)) (get-in stats [:emitted k]) (get-in stats [:transferred k]) (float-str (get-in stats [:complete-latencies k])) @@ -717,7 +717,7 @@ :let [disp ((display-map k) k)]] [(link-to (if (= k window) {:class "red"} {}) (url-format "/topology/%s/component/%s?window=%s" topology-id id k) - disp) + (escape-html disp)) (get-in stats [:emitted k]) (get-in stats [:transferred k]) (float-str (get-in stats [:complete-latencies k])) @@ -935,7 +935,7 @@ :let [disp ((display-map k) k)]] [(link-to (if (= k window) {:class "red"} {}) (url-format "/topology/%s/component/%s?window=%s" topology-id id k) - disp) + (escape-html disp)) (get-in stats [:emitted k]) (get-in stats [:transferred k]) (float-str (get-in stats [:execute-latencies k])) From 2c636ba017a8b67eceb11a8b09caf081812752b0 Mon Sep 17 00:00:00 2001 From: Niels Basjes Date: Mon, 24 Mar 2014 22:07:27 +0100 Subject: [PATCH 309/349] Ensure that also the topology id is encoded/decoded correctly --- storm-core/src/clj/backtype/storm/ui/core.clj | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj index 3b975c5a62..ad4b97fa91 100644 --- a/storm-core/src/clj/backtype/storm/ui/core.clj +++ b/storm-core/src/clj/backtype/storm/ui/core.clj @@ -1016,34 +1016,40 @@ (-> (main-page) ui-template)) (GET "/topology/:id" [:as {cookies :cookies} id & m] - (let [include-sys? (get-include-sys? cookies)] + (let [include-sys? (get-include-sys? cookies) + id (java.net.URLDecoder/decode id)] (try (-> (topology-page id (:window m) include-sys?) (concat [(mk-system-toggle-button include-sys?)]) ui-template) (catch Exception e (resp/redirect "/"))))) (GET "/topology/:id/component/:component" [:as {cookies :cookies} id component & m] - (let [include-sys? (get-include-sys? cookies)] - (-> (component-page id (java.net.URLDecoder/decode component) (:window m) include-sys?) + (let [include-sys? (get-include-sys? cookies) + id (java.net.URLDecoder/decode id) + component (java.net.URLDecoder/decode component)] + (-> (component-page id component (:window m) include-sys?) (concat [(mk-system-toggle-button include-sys?)]) ui-template))) (POST "/topology/:id/activate" [id] (with-nimbus nimbus - (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id) + (let [id (java.net.URLDecoder/decode id) + tplg (.getTopologyInfo ^Nimbus$Client nimbus id) name (.get_name tplg)] (.activate nimbus name) (log-message "Activating topology '" name "'"))) (resp/redirect (str "/topology/" id))) (POST "/topology/:id/deactivate" [id] (with-nimbus nimbus - (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id) + (let [id (java.net.URLDecoder/decode id) + tplg (.getTopologyInfo ^Nimbus$Client nimbus id) name (.get_name tplg)] (.deactivate nimbus name) (log-message "Deactivating topology '" name "'"))) (resp/redirect (str "/topology/" id))) (POST "/topology/:id/rebalance/:wait-time" [id wait-time] (with-nimbus nimbus - (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id) + (let [id (java.net.URLDecoder/decode id) + tplg (.getTopologyInfo ^Nimbus$Client nimbus id) name (.get_name tplg) options (RebalanceOptions.)] (.set_wait_secs options (Integer/parseInt wait-time)) @@ -1052,7 +1058,8 @@ (resp/redirect (str "/topology/" id))) (POST "/topology/:id/kill/:wait-time" [id wait-time] (with-nimbus nimbus - (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id) + (let [id (java.net.URLDecoder/decode id) + tplg (.getTopologyInfo ^Nimbus$Client nimbus id) name (.get_name tplg) options (KillOptions.)] (.set_wait_secs options (Integer/parseInt wait-time)) From 447fdb0b56757cd8d97d7c684f8a0b3b68639673 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 26 Mar 2014 12:32:47 -0400 Subject: [PATCH 310/349] STORM-263: update to carbonite 1.4.0/kryo 2.21 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 044d5e7893..e4129ed873 100644 --- a/pom.xml +++ b/pom.xml @@ -162,7 +162,7 @@ 0.3.11 0.2.3 0.0.1 - 1.3.2 + 1.4.0 1.11 4.1.1 0.2.2 From 2f45866c8e011ac4804c940ff9e1d7c147591761 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Mon, 31 Mar 2014 07:35:22 +0100 Subject: [PATCH 311/349] fixed KafkaSpout partition assignment * added partitions -> task mapping logging --- CHANGELOG.md | 2 + src/jvm/storm/kafka/KafkaUtils.java | 31 ++++- src/jvm/storm/kafka/StaticCoordinator.java | 7 +- src/jvm/storm/kafka/ZkCoordinator.java | 37 +++--- src/test/storm/kafka/KafkaUtilsTest.java | 39 ++++++ src/test/storm/kafka/TestUtils.java | 20 +++ src/test/storm/kafka/ZkCoordinatorTest.java | 128 ++++++++++++++++++++ 7 files changed, 238 insertions(+), 26 deletions(-) create mode 100644 src/test/storm/kafka/TestUtils.java create mode 100644 src/test/storm/kafka/ZkCoordinatorTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 74ab824c12..133ffdaaff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 0.5.0 +* fixed partition assignment for KafkaSpout ## 0.4.0 * added support for reading kafka message keys * configurable metrics emit interval diff --git a/src/jvm/storm/kafka/KafkaUtils.java b/src/jvm/storm/kafka/KafkaUtils.java index eed438f5dc..4e8b3a3b1d 100644 --- a/src/jvm/storm/kafka/KafkaUtils.java +++ b/src/jvm/storm/kafka/KafkaUtils.java @@ -2,6 +2,7 @@ import backtype.storm.metric.api.IMetric; import backtype.storm.utils.Utils; +import com.google.common.base.Preconditions; import kafka.api.FetchRequest; import kafka.api.FetchRequestBuilder; import kafka.api.PartitionOffsetRequestInfo; @@ -13,6 +14,7 @@ import kafka.message.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import storm.kafka.trident.GlobalPartitionInformation; import storm.kafka.trident.IBrokerReader; import storm.kafka.trident.StaticBrokerReader; import storm.kafka.trident.ZkBrokerReader; @@ -39,7 +41,7 @@ public static IBrokerReader makeBrokerReader(Map stormConf, KafkaConfig conf) { public static long getOffset(SimpleConsumer consumer, String topic, int partition, KafkaConfig config) { long startOffsetTime = kafka.api.OffsetRequest.LatestTime(); - if ( config.forceFromStart ) { + if (config.forceFromStart) { startOffsetTime = config.startOffsetTime; } return getOffset(consumer, topic, partition, startOffsetTime); @@ -91,7 +93,7 @@ public Object getValueAndReset() { LOG.warn("partitionToOffset contains partition not found in _connections. Stale partition data?"); return null; } - long earliestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.EarliestTime()); + long earliestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.EarliestTime()); long latestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.LatestTime()); if (earliestTimeOffset == 0 || latestTimeOffset == 0) { LOG.warn("No data found in Kafka Partition " + partition.getId()); @@ -184,4 +186,29 @@ public static Iterable> generateTuples(KafkaConfig kafkaConfig, Mes return tups; } + + public static List calculatePartitionsForTask(GlobalPartitionInformation partitionInformation, int totalTasks, int taskIndex) { + Preconditions.checkArgument(taskIndex < totalTasks, "task index must be less that total tasks"); + List partitions = partitionInformation.getOrderedPartitions(); + int numPartitions = partitions.size(); + if (numPartitions < totalTasks) { + LOG.warn("there are more tasks than partitions (tasks: " + totalTasks + "; partitions: " + numPartitions + "), some tasks will be idle"); + } + List taskPartitions = new ArrayList(); + for (int i = taskIndex; i < numPartitions; i += totalTasks) { + Partition taskPartition = partitions.get(i); + taskPartitions.add(taskPartition); + } + logPartitionMapping(totalTasks, taskIndex, taskPartitions); + return taskPartitions; + } + + private static void logPartitionMapping(int totalTasks, int taskIndex, List taskPartitions) { + String taskPrefix = "[" + taskIndex + "/" + totalTasks + "] --> "; + if (taskPartitions.isEmpty()) { + LOG.warn(taskPrefix + "no partitions assigned"); + } else { + LOG.info(taskPrefix + "assigned " + taskPartitions); + } + } } diff --git a/src/jvm/storm/kafka/StaticCoordinator.java b/src/jvm/storm/kafka/StaticCoordinator.java index 741552242b..040060c524 100644 --- a/src/jvm/storm/kafka/StaticCoordinator.java +++ b/src/jvm/storm/kafka/StaticCoordinator.java @@ -12,13 +12,10 @@ public class StaticCoordinator implements PartitionCoordinator { public StaticCoordinator(DynamicPartitionConnections connections, Map stormConf, SpoutConfig config, ZkState state, int taskIndex, int totalTasks, String topologyInstanceId) { StaticHosts hosts = (StaticHosts) config.hosts; - List partitions = hosts.getPartitionInformation().getOrderedPartitions(); - for (int i = taskIndex; i < partitions.size(); i += totalTasks) { - Partition myPartition = partitions.get(i); + List myPartitions = KafkaUtils.calculatePartitionsForTask(hosts.getPartitionInformation(), totalTasks, taskIndex); + for (Partition myPartition : myPartitions) { _managers.put(myPartition, new PartitionManager(connections, topologyInstanceId, state, stormConf, config, myPartition)); - } - _allManagers = new ArrayList(_managers.values()); } diff --git a/src/jvm/storm/kafka/ZkCoordinator.java b/src/jvm/storm/kafka/ZkCoordinator.java index 98e51a3170..35d2c57ea7 100644 --- a/src/jvm/storm/kafka/ZkCoordinator.java +++ b/src/jvm/storm/kafka/ZkCoordinator.java @@ -1,6 +1,5 @@ package storm.kafka; -import backtype.storm.task.IMetricsContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import storm.kafka.trident.GlobalPartitionInformation; @@ -22,9 +21,12 @@ public class ZkCoordinator implements PartitionCoordinator { DynamicBrokersReader _reader; ZkState _state; Map _stormConf; - IMetricsContext _metricsContext; public ZkCoordinator(DynamicPartitionConnections connections, Map stormConf, SpoutConfig spoutConfig, ZkState state, int taskIndex, int totalTasks, String topologyInstanceId) { + this(connections, stormConf, spoutConfig, state, taskIndex, totalTasks, topologyInstanceId, buildReader(stormConf, spoutConfig)); + } + + public ZkCoordinator(DynamicPartitionConnections connections, Map stormConf, SpoutConfig spoutConfig, ZkState state, int taskIndex, int totalTasks, String topologyInstanceId, DynamicBrokersReader reader) { _spoutConfig = spoutConfig; _connections = connections; _taskIndex = taskIndex; @@ -32,11 +34,14 @@ public ZkCoordinator(DynamicPartitionConnections connections, Map stormConf, Spo _topologyInstanceId = topologyInstanceId; _stormConf = stormConf; _state = state; - ZkHosts brokerConf = (ZkHosts) spoutConfig.hosts; _refreshFreqMs = brokerConf.refreshFreqSecs * 1000; - _reader = new DynamicBrokersReader(stormConf, brokerConf.brokerZkStr, brokerConf.brokerZkPath, spoutConfig.topic); + _reader = reader; + } + private static DynamicBrokersReader buildReader(Map stormConf, SpoutConfig spoutConfig) { + ZkHosts hosts = (ZkHosts) spoutConfig.hosts; + return new DynamicBrokersReader(stormConf, hosts.brokerZkStr, hosts.brokerZkPath, spoutConfig.topic); } @Override @@ -50,14 +55,9 @@ public List getMyManagedPartitions() { void refresh() { try { - LOG.info("Refreshing partition manager connections"); + LOG.info(taskIdentifier() + "Refreshing partition manager connections"); GlobalPartitionInformation brokerInfo = _reader.getBrokerInfo(); - Set mine = new HashSet(); - for (Partition partitionId : brokerInfo) { - if (myOwnership(partitionId)) { - mine.add(partitionId); - } - } + List mine = KafkaUtils.calculatePartitionsForTask(brokerInfo, _totalTasks, _taskIndex); Set curr = _managers.keySet(); Set newPartitions = new HashSet(mine); @@ -66,13 +66,13 @@ void refresh() { Set deletedPartitions = new HashSet(curr); deletedPartitions.removeAll(mine); - LOG.info("Deleted partition managers: " + deletedPartitions.toString()); + LOG.info(taskIdentifier() + "Deleted partition managers: " + deletedPartitions.toString()); for (Partition id : deletedPartitions) { PartitionManager man = _managers.remove(id); man.close(); } - LOG.info("New partition managers: " + newPartitions.toString()); + LOG.info(taskIdentifier() + "New partition managers: " + newPartitions.toString()); for (Partition id : newPartitions) { PartitionManager man = new PartitionManager(_connections, _topologyInstanceId, _state, _stormConf, _spoutConfig, id); @@ -83,16 +83,15 @@ void refresh() { throw new RuntimeException(e); } _cachedList = new ArrayList(_managers.values()); - LOG.info("Finished refreshing"); + LOG.info(taskIdentifier() + "Finished refreshing"); + } + + private String taskIdentifier() { + return "[" + _taskIndex + "/" + _totalTasks + "] - "; } @Override public PartitionManager getManager(Partition partition) { return _managers.get(partition); } - - private boolean myOwnership(Partition id) { - int val = Math.abs(id.host.hashCode() + 23 * id.partition); - return val % _totalTasks == _taskIndex; - } } diff --git a/src/test/storm/kafka/KafkaUtilsTest.java b/src/test/storm/kafka/KafkaUtilsTest.java index 07630427cf..a4e7f52f1a 100644 --- a/src/test/storm/kafka/KafkaUtilsTest.java +++ b/src/test/storm/kafka/KafkaUtilsTest.java @@ -178,4 +178,43 @@ private void createTopicAndSendMessage(String key, String value) { Producer producer = new Producer(producerConfig); producer.send(new KeyedMessage(config.topic, key, value)); } + + + @Test + public void assignOnePartitionPerTask() { + runPartitionToTaskMappingTest(16, 1); + } + + @Test + public void assignTwoPartitionsPerTask() { + runPartitionToTaskMappingTest(16, 2); + } + + @Test + public void assignAllPartitionsToOneTask() { + runPartitionToTaskMappingTest(32, 32); + } + + + public void runPartitionToTaskMappingTest(int numPartitions, int partitionsPerTask) { + GlobalPartitionInformation globalPartitionInformation = TestUtils.buildPartitionInfo(numPartitions); + int numTasks = numPartitions / partitionsPerTask; + for (int i = 0 ; i < numTasks ; i++) { + assertEquals(partitionsPerTask, KafkaUtils.calculatePartitionsForTask(globalPartitionInformation, numTasks, i).size()); + } + } + + @Test + public void moreTasksThanPartitions() { + GlobalPartitionInformation globalPartitionInformation = TestUtils.buildPartitionInfo(1); + int numTasks = 2; + assertEquals(1, KafkaUtils.calculatePartitionsForTask(globalPartitionInformation, numTasks, 0).size()); + assertEquals(0, KafkaUtils.calculatePartitionsForTask(globalPartitionInformation, numTasks, 1).size()); + } + + @Test (expected = IllegalArgumentException.class ) + public void assignInvalidTask() { + GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(); + KafkaUtils.calculatePartitionsForTask(globalPartitionInformation, 1, 1); + } } diff --git a/src/test/storm/kafka/TestUtils.java b/src/test/storm/kafka/TestUtils.java new file mode 100644 index 0000000000..860d96d605 --- /dev/null +++ b/src/test/storm/kafka/TestUtils.java @@ -0,0 +1,20 @@ +package storm.kafka; + +import storm.kafka.trident.GlobalPartitionInformation; + +public class TestUtils { + + public static GlobalPartitionInformation buildPartitionInfo(int numPartitions) { + return buildPartitionInfo(numPartitions, 9092); + } + + + public static GlobalPartitionInformation buildPartitionInfo(int numPartitions, int brokerPort) { + GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(); + for (int i = 0; i < numPartitions; i++) { + globalPartitionInformation.addPartition(i, Broker.fromString("broker-" + i + " :" + brokerPort)); + } + return globalPartitionInformation; + } + +} diff --git a/src/test/storm/kafka/ZkCoordinatorTest.java b/src/test/storm/kafka/ZkCoordinatorTest.java new file mode 100644 index 0000000000..35b3b4b3f1 --- /dev/null +++ b/src/test/storm/kafka/ZkCoordinatorTest.java @@ -0,0 +1,128 @@ +package storm.kafka; + +import backtype.storm.Config; +import com.netflix.curator.test.TestingServer; +import kafka.javaapi.consumer.SimpleConsumer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Mockito.when; + +public class ZkCoordinatorTest { + + + @Mock + private DynamicBrokersReader reader; + + @Mock + private DynamicPartitionConnections dynamicPartitionConnections; + + private KafkaTestBroker broker = new KafkaTestBroker(); + private TestingServer server; + private Map stormConf = new HashMap(); + private SpoutConfig spoutConfig; + private ZkState state; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + server = new TestingServer(); + String connectionString = server.getConnectString(); + ZkHosts hosts = new ZkHosts(connectionString); + hosts.refreshFreqSecs = 1; + spoutConfig = new SpoutConfig(hosts, "topic", "/test", "id"); + Map conf = buildZookeeperConfig(server); + state = new ZkState(conf); + SimpleConsumer simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient"); + when(dynamicPartitionConnections.register(any(Broker.class), anyInt())).thenReturn(simpleConsumer); + } + + private Map buildZookeeperConfig(TestingServer server) { + Map conf = new HashMap(); + conf.put(Config.TRANSACTIONAL_ZOOKEEPER_PORT, server.getPort()); + conf.put(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS, Arrays.asList("localhost")); + conf.put(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT, 20000); + conf.put(Config.STORM_ZOOKEEPER_RETRY_TIMES, 3); + conf.put(Config.STORM_ZOOKEEPER_RETRY_INTERVAL, 30); + return conf; + } + + @After + public void shutdown() throws Exception { + broker.shutdown(); + server.stop(); + } + + @Test + public void testOnePartitionPerTask() throws Exception { + int totalTasks = 64; + int partitionsPerTask = 1; + List coordinatorList = buildCoordinators(totalTasks / partitionsPerTask); + when(reader.getBrokerInfo()).thenReturn(TestUtils.buildPartitionInfo(totalTasks)); + for (ZkCoordinator coordinator : coordinatorList) { + List myManagedPartitions = coordinator.getMyManagedPartitions(); + assertEquals(partitionsPerTask, myManagedPartitions.size()); + assertEquals(coordinator._taskIndex, myManagedPartitions.get(0).getPartition().partition); + } + } + + + @Test + public void testPartitionsChange() throws Exception { + final int totalTasks = 64; + int partitionsPerTask = 2; + List coordinatorList = buildCoordinators(totalTasks / partitionsPerTask); + when(reader.getBrokerInfo()).thenReturn(TestUtils.buildPartitionInfo(totalTasks, 9092)); + List> partitionManagersBeforeRefresh = getPartitionManagers(coordinatorList); + waitForRefresh(); + when(reader.getBrokerInfo()).thenReturn(TestUtils.buildPartitionInfo(totalTasks, 9093)); + List> partitionManagersAfterRefresh = getPartitionManagers(coordinatorList); + assertEquals(partitionManagersAfterRefresh.size(), partitionManagersAfterRefresh.size()); + Iterator> iterator = partitionManagersAfterRefresh.iterator(); + for (List partitionManagersBefore : partitionManagersBeforeRefresh) { + List partitionManagersAfter = iterator.next(); + assertPartitionsAreDifferent(partitionManagersBefore, partitionManagersAfter, partitionsPerTask); + } + } + + private void assertPartitionsAreDifferent(List partitionManagersBefore, List partitionManagersAfter, int partitionsPerTask) { + assertEquals(partitionsPerTask, partitionManagersBefore.size()); + assertEquals(partitionManagersBefore.size(), partitionManagersAfter.size()); + for (int i = 0; i < partitionsPerTask; i++) { + assertNotEquals(partitionManagersBefore.get(i).getPartition(), partitionManagersAfter.get(i).getPartition()); + } + + } + + private List> getPartitionManagers(List coordinatorList) { + List> partitions = new ArrayList(); + for (ZkCoordinator coordinator : coordinatorList) { + partitions.add(coordinator.getMyManagedPartitions()); + } + return partitions; + } + + private void waitForRefresh() throws InterruptedException { + Thread.sleep(((ZkHosts) spoutConfig.hosts).refreshFreqSecs * 1000 + 1); + } + + private List buildCoordinators(int totalTasks) { + List coordinatorList = new ArrayList(); + for (int i = 0; i < totalTasks; i++) { + ZkCoordinator coordinator = new ZkCoordinator(dynamicPartitionConnections, stormConf, spoutConfig, state, i, totalTasks, "test-id", reader); + coordinatorList.add(coordinator); + } + return coordinatorList; + } + + +} From 9129536cd91ea2b54df5ba2a4eb524aa064c56a1 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Mon, 31 Mar 2014 22:48:08 +0100 Subject: [PATCH 312/349] upgraded storm to 0.9.1 --- CHANGELOG.md | 1 + pom.xml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 133ffdaaff..f87642148e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## 0.5.0 * fixed partition assignment for KafkaSpout +* upgraded to storm 0.9.1 ## 0.4.0 * added support for reading kafka message keys * configurable metrics emit interval diff --git a/pom.xml b/pom.xml index ff9b9c1a38..4626fbd8f7 100644 --- a/pom.xml +++ b/pom.xml @@ -174,9 +174,9 @@ - storm - storm - 0.9.0 + org.apache.storm + storm-core + 0.9.1-incubating provided From 94c4d4d9e6c4ce736141668c585818214a9d26cf Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Tue, 1 Apr 2014 15:32:31 +0000 Subject: [PATCH 313/349] STORM-12 Reduce thread usage of Netty transport. --- .../storm/messaging/netty/Client.java | 184 ++++++++++++------ .../storm/messaging/netty/Context.java | 30 ++- .../storm/messaging/netty/MessageBatch.java | 55 +----- .../storm/messaging/netty/Server.java | 2 +- .../messaging/netty/StormClientHandler.java | 80 +++----- 5 files changed, 183 insertions(+), 168 deletions(-) diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java b/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java index f15cd1d412..6996b49cfa 100644 --- a/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java +++ b/storm-core/src/jvm/backtype/storm/messaging/netty/Client.java @@ -21,17 +21,21 @@ import backtype.storm.messaging.IConnection; import backtype.storm.messaging.TaskMessage; import backtype.storm.utils.Utils; + import org.jboss.netty.bootstrap.ClientBootstrap; import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.ChannelFactory; import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory; +import org.jboss.netty.channel.ChannelFuture; +import org.jboss.netty.channel.ChannelFutureListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; import java.util.Random; -import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -39,38 +43,37 @@ class Client implements IConnection { private static final Logger LOG = LoggerFactory.getLogger(Client.class); + private static final Timer TIMER = new Timer("netty-client-timer", true); + private final int max_retries; private final long base_sleep_ms; private final long max_sleep_ms; private LinkedBlockingQueue message_queue; //entry should either be TaskMessage or ControlMessage private AtomicReference channelRef; private final ClientBootstrap bootstrap; - private InetSocketAddress remote_addr; + InetSocketAddress remote_addr; private AtomicInteger retries; private final Random random = new Random(); private final ChannelFactory factory; private final int buffer_size; private final AtomicBoolean being_closed; + private boolean wait_for_requests; @SuppressWarnings("rawtypes") - Client(Map storm_conf, String host, int port) { + Client(Map storm_conf, ChannelFactory factory, String host, int port) { + this.factory = factory; message_queue = new LinkedBlockingQueue(); retries = new AtomicInteger(0); channelRef = new AtomicReference(null); being_closed = new AtomicBoolean(false); + wait_for_requests = false; // Configure buffer_size = Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_BUFFER_SIZE)); max_retries = Math.min(30, Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_MAX_RETRIES))); base_sleep_ms = Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_MIN_SLEEP_MS)); max_sleep_ms = Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_MAX_SLEEP_MS)); - int maxWorkers = Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_CLIENT_WORKER_THREADS)); - if (maxWorkers > 0) { - factory = new NioClientSocketChannelFactory(Executors.newCachedThreadPool(), Executors.newCachedThreadPool(), maxWorkers); - } else { - factory = new NioClientSocketChannelFactory(Executors.newCachedThreadPool(), Executors.newCachedThreadPool()); - } bootstrap = new ClientBootstrap(factory); bootstrap.setOption("tcpNoDelay", true); bootstrap.setOption("sendBufferSize", buffer_size); @@ -88,19 +91,24 @@ class Client implements IConnection { * We will retry connection with exponential back-off policy */ void reconnect() { - try { - int tried_count = retries.incrementAndGet(); - if (tried_count <= max_retries) { - Thread.sleep(getSleepTimeMs()); - LOG.info("Reconnect ... [{}]", tried_count); - bootstrap.connect(remote_addr); - LOG.debug("connection started..."); - } else { - LOG.warn("Remote address is not reachable. We will close this client."); - close(); - } - } catch (InterruptedException e) { - LOG.warn("connection failed", e); + close_n_release(); + + //reconnect only if it's not being closed + if (being_closed.get()) return; + + final int tried_count = retries.incrementAndGet(); + if (tried_count <= max_retries) { + long sleep = getSleepTimeMs(); + LOG.info("Waiting {} ms before trying connection to {}", sleep, remote_addr); + TIMER.schedule(new TimerTask() { + @Override + public void run() { + LOG.info("Reconnect ... [{}] to {}", tried_count, remote_addr); + bootstrap.connect(remote_addr); + }}, sleep); + } else { + LOG.warn(remote_addr+" is not reachable. We will close this client."); + close(); } } @@ -130,36 +138,94 @@ public void send(int task, byte[] message) { try { message_queue.put(new TaskMessage(task, message)); + + //resume delivery if it is waiting for requests + tryDeliverMessages(true); } catch (InterruptedException e) { throw new RuntimeException(e); } } + /** + * Retrieve messages from queue, and delivery to server if any + */ + synchronized void tryDeliverMessages(boolean only_if_waiting) throws InterruptedException { + //just skip if delivery only if waiting, and we are not waiting currently + if (only_if_waiting && !wait_for_requests) return; + + //make sure that channel was not closed + Channel channel = channelRef.get(); + if (channel == null) return; + if (!channel.isOpen()) { + LOG.info("Channel to {} is no longer open.",remote_addr); + //The channel is not open yet. Reconnect? + reconnect(); + return; + } + + final MessageBatch requests = tryTakeMessages(); + if (requests==null) { + wait_for_requests = true; + return; + } + + //if channel is being closed and we have no outstanding messages, let's close the channel + if (requests.isEmpty() && being_closed.get()) { + close_n_release(); + return; + } + + //we are busily delivering messages, and will check queue upon response. + //When send() is called by senders, we should not thus call tryDeliverMessages(). + wait_for_requests = false; + + //write request into socket channel + ChannelFuture future = channel.write(requests); + future.addListener(new ChannelFutureListener() { + public void operationComplete(ChannelFuture future) + throws Exception { + if (!future.isSuccess()) { + LOG.info("failed to send "+requests.size()+" requests to "+remote_addr, future.getCause()); + reconnect(); + } else { + LOG.debug("{} request(s) sent", requests.size()); + + //Now that our requests have been sent, channel could be closed if needed + if (being_closed.get()) + close_n_release(); + } + } + }); + } + /** * Take all enqueued messages from queue - * @return + * @return batch of messages * @throws InterruptedException + * + * synchronized ... ensure that messages are delivered in the same order + * as they are added into queue */ - MessageBatch takeMessages() throws InterruptedException { + private MessageBatch tryTakeMessages() throws InterruptedException { //1st message - MessageBatch batch = new MessageBatch(buffer_size); - Object msg = message_queue.take(); - batch.add(msg); + Object msg = message_queue.poll(); + if (msg == null) return null; + MessageBatch batch = new MessageBatch(buffer_size); //we will discard any message after CLOSE - if (msg==ControlMessage.CLOSE_MESSAGE) + if (msg == ControlMessage.CLOSE_MESSAGE) { + LOG.info("Connection to {} is being closed", remote_addr); + being_closed.set(true); return batch; + } - while (!batch.isFull()) { - //peek the next message - msg = message_queue.peek(); - //no more messages - if (msg == null) break; - - //we will discard any message after CLOSE - if (msg==ControlMessage.CLOSE_MESSAGE) { + batch.add((TaskMessage)msg); + while (!batch.isFull() && ((msg = message_queue.peek())!=null)) { + //Is it a CLOSE message? + if (msg == ControlMessage.CLOSE_MESSAGE) { message_queue.take(); - batch.add(msg); + LOG.info("Connection to {} is being closed", remote_addr); + being_closed.set(true); break; } @@ -179,31 +245,29 @@ MessageBatch takeMessages() throws InterruptedException { * * We will send all existing requests, and then invoke close_n_release() method */ - public synchronized void close() { - if (!being_closed.get()) { - //enqueue a CLOSE message so that shutdown() will be invoked - try { - message_queue.put(ControlMessage.CLOSE_MESSAGE); - being_closed.set(true); - } catch (InterruptedException e) { - close_n_release(); - } + public void close() { + //enqueue a CLOSE message so that shutdown() will be invoked + try { + message_queue.put(ControlMessage.CLOSE_MESSAGE); + + //resume delivery if it is waiting for requests + tryDeliverMessages(true); + } catch (InterruptedException e) { + LOG.info("Interrupted Connection to {} is being closed", remote_addr); + being_closed.set(true); + close_n_release(); } } /** * close_n_release() is invoked after all messages have been sent. */ - void close_n_release() { - if (channelRef.get() != null) - channelRef.get().close().awaitUninterruptibly(); - - //we need to release resources - new Thread(new Runnable() { - @Override - public void run() { - factory.releaseExternalResources(); - }}).start(); + synchronized void close_n_release() { + if (channelRef.get() != null) { + channelRef.get().close(); + LOG.debug("channel {} closed",remote_addr); + setChannel(null); + } } public TaskMessage recv(int flags) { @@ -211,6 +275,10 @@ public TaskMessage recv(int flags) { } void setChannel(Channel channel) { + if (channel != null && channel.isOpen()) { + //Assume the most recent connection attempt was successful. + retries.set(0); + } channelRef.set(channel); //reset retries if (channel != null) @@ -218,7 +286,3 @@ void setChannel(Channel channel) { } } - - - - diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/Context.java b/storm-core/src/jvm/backtype/storm/messaging/netty/Context.java index 3e09dd1c8a..80b4443e1c 100644 --- a/storm-core/src/jvm/backtype/storm/messaging/netty/Context.java +++ b/storm-core/src/jvm/backtype/storm/messaging/netty/Context.java @@ -17,8 +17,16 @@ */ package backtype.storm.messaging.netty; +import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory; + +import java.util.concurrent.Executors; +import java.util.Map; +import java.util.Vector; + +import backtype.storm.Config; import backtype.storm.messaging.IConnection; import backtype.storm.messaging.IContext; +import backtype.storm.utils.Utils; import java.util.Map; import java.util.Vector; @@ -27,14 +35,25 @@ public class Context implements IContext { @SuppressWarnings("rawtypes") private Map storm_conf; private volatile Vector connections; - + private NioClientSocketChannelFactory clientChannelFactory; + /** * initialization per Storm configuration */ @SuppressWarnings("rawtypes") public void prepare(Map storm_conf) { - this.storm_conf = storm_conf; - connections = new Vector(); + this.storm_conf = storm_conf; + connections = new Vector(); + + //each context will have a single client channel factory + int maxWorkers = Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_CLIENT_WORKER_THREADS)); + if (maxWorkers > 0) { + clientChannelFactory = new NioClientSocketChannelFactory(Executors.newCachedThreadPool(), + Executors.newCachedThreadPool(), maxWorkers); + } else { + clientChannelFactory = new NioClientSocketChannelFactory(Executors.newCachedThreadPool(), + Executors.newCachedThreadPool()); + } } /** @@ -50,7 +69,7 @@ public IConnection bind(String storm_id, int port) { * establish a connection to a remote server */ public IConnection connect(String storm_id, String host, int port) { - IConnection client = new Client(storm_conf, host, port); + IConnection client = new Client(storm_conf, clientChannelFactory, host, port); connections.add(client); return client; } @@ -63,5 +82,8 @@ public void term() { conn.close(); } connections = null; + + //we need to release resources associated with client channel factory + clientChannelFactory.releaseExternalResources(); } } diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/MessageBatch.java b/storm-core/src/jvm/backtype/storm/messaging/netty/MessageBatch.java index 9d287e4b5f..cd8d4e378a 100644 --- a/storm-core/src/jvm/backtype/storm/messaging/netty/MessageBatch.java +++ b/storm-core/src/jvm/backtype/storm/messaging/netty/MessageBatch.java @@ -26,56 +26,22 @@ class MessageBatch { private int buffer_size; - private ArrayList msgs; + private ArrayList msgs; private int encoded_length; MessageBatch(int buffer_size) { this.buffer_size = buffer_size; - msgs = new ArrayList(); + msgs = new ArrayList(); encoded_length = ControlMessage.EOB_MESSAGE.encodeLength(); } - void add(Object obj) { + void add(TaskMessage obj) { if (obj == null) throw new RuntimeException("null object forbidded in message batch"); - if (obj instanceof TaskMessage) { - TaskMessage msg = (TaskMessage)obj; - msgs.add(msg); - encoded_length += msgEncodeLength(msg); - return; - } - - if (obj instanceof ControlMessage) { - ControlMessage msg = (ControlMessage)obj; - msgs.add(msg); - encoded_length += msg.encodeLength(); - return; - } - - throw new RuntimeException("Unsuppoted object type "+obj.getClass().getName()); - } - - void remove(Object obj) { - if (obj == null) return; - - if (obj instanceof TaskMessage) { - TaskMessage msg = (TaskMessage)obj; - msgs.remove(msg); - encoded_length -= msgEncodeLength(msg); - return; - } - - if (obj instanceof ControlMessage) { - ControlMessage msg = (ControlMessage)obj; - msgs.remove(msg); - encoded_length -= msg.encodeLength(); - return; - } - } - - Object get(int index) { - return msgs.get(index); + TaskMessage msg = (TaskMessage)obj; + msgs.add(msg); + encoded_length += msgEncodeLength(msg); } /** @@ -129,12 +95,9 @@ int size() { ChannelBuffer buffer() throws Exception { ChannelBufferOutputStream bout = new ChannelBufferOutputStream(ChannelBuffers.directBuffer(encoded_length)); - for (Object msg : msgs) - if (msg instanceof TaskMessage) - writeTaskMessage(bout, (TaskMessage)msg); - else - ((ControlMessage)msg).write(bout); - + for (TaskMessage msg : msgs) + writeTaskMessage(bout, msg); + //add a END_OF_BATCH indicator ControlMessage.EOB_MESSAGE.write(bout); diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/Server.java b/storm-core/src/jvm/backtype/storm/messaging/netty/Server.java index ad811b0622..83e418767a 100644 --- a/storm-core/src/jvm/backtype/storm/messaging/netty/Server.java +++ b/storm-core/src/jvm/backtype/storm/messaging/netty/Server.java @@ -123,7 +123,7 @@ protected void closeChannel(Channel channel) { * close all channels, and release resources */ public synchronized void close() { - if (allChannels != null) { + if (allChannels != null) { allChannels.close().awaitUninterruptibly(); factory.releaseExternalResources(); allChannels = null; diff --git a/storm-core/src/jvm/backtype/storm/messaging/netty/StormClientHandler.java b/storm-core/src/jvm/backtype/storm/messaging/netty/StormClientHandler.java index 65c36a7c59..43a8c39123 100644 --- a/storm-core/src/jvm/backtype/storm/messaging/netty/StormClientHandler.java +++ b/storm-core/src/jvm/backtype/storm/messaging/netty/StormClientHandler.java @@ -17,7 +17,15 @@ */ package backtype.storm.messaging.netty; -import org.jboss.netty.channel.*; +import java.net.ConnectException; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.jboss.netty.channel.Channel; +import org.jboss.netty.channel.ChannelHandlerContext; +import org.jboss.netty.channel.ChannelStateEvent; +import org.jboss.netty.channel.ExceptionEvent; +import org.jboss.netty.channel.MessageEvent; +import org.jboss.netty.channel.SimpleChannelUpstreamHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,12 +35,10 @@ public class StormClientHandler extends SimpleChannelUpstreamHandler { private static final Logger LOG = LoggerFactory.getLogger(StormClientHandler.class); private Client client; - private AtomicBoolean being_closed; - long start_time; + long start_time; StormClientHandler(Client client) { this.client = client; - being_closed = new AtomicBoolean(false); start_time = System.currentTimeMillis(); } @@ -41,13 +47,14 @@ public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent event) //register the newly established channel Channel channel = event.getChannel(); client.setChannel(channel); - LOG.debug("connection established to a remote host"); + LOG.info("connection established from "+channel.getLocalAddress()+" to "+channel.getRemoteAddress()); - //send next request + //send next batch of requests if any try { - sendRequests(channel, client.takeMessages()); - } catch (InterruptedException e) { - channel.close(); + client.tryDeliverMessages(false); + } catch (Exception ex) { + LOG.info("exception when sending messages:", ex.getMessage()); + client.reconnect(); } } @@ -60,62 +67,21 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent event) { if (msg==ControlMessage.FAILURE_RESPONSE) LOG.info("failure response:{}", msg); - //send next request - Channel channel = event.getChannel(); + //send next batch of requests if any try { - sendRequests(channel, client.takeMessages()); - } catch (InterruptedException e) { - channel.close(); - } - } - - /** - * Retrieve a request from message queue, and send to server - * @param channel - */ - private void sendRequests(Channel channel, final MessageBatch requests) { - if (requests==null || requests.size()==0 || being_closed.get()) return; - - //if task==CLOSE_MESSAGE for our last request, the channel is to be closed - Object last_msg = requests.get(requests.size()-1); - if (last_msg==ControlMessage.CLOSE_MESSAGE) { - being_closed.set(true); - requests.remove(last_msg); - } - - //we may don't need do anything if no requests found - if (requests.isEmpty()) { - if (being_closed.get()) - client.close_n_release(); - return; + client.tryDeliverMessages(false); + } catch (Exception ex) { + LOG.info("exception when sending messages:", ex.getMessage()); + client.reconnect(); } - - //write request into socket channel - ChannelFuture future = channel.write(requests); - future.addListener(new ChannelFutureListener() { - public void operationComplete(ChannelFuture future) - throws Exception { - if (!future.isSuccess()) { - LOG.info("failed to send requests:", future.getCause()); - future.getChannel().close(); - } else { - LOG.debug("{} request(s) sent", requests.size()); - } - if (being_closed.get()) - client.close_n_release(); - } - }); } @Override public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent event) { Throwable cause = event.getCause(); if (!(cause instanceof ConnectException)) { - LOG.info("Connection failed:", cause); - } - if (!being_closed.get()) { - client.setChannel(null); - client.reconnect(); + LOG.info("Connection to "+client.remote_addr+" failed:", cause); } + client.reconnect(); } } From ae71186acdeb5a7efc19393d10cdde997f587773 Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Thu, 3 Apr 2014 17:29:32 +0000 Subject: [PATCH 314/349] STORM-270 don't package .clj files in release jars. --- storm-core/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/storm-core/pom.xml b/storm-core/pom.xml index c5baad0ed4..8eee3a9945 100644 --- a/storm-core/pom.xml +++ b/storm-core/pom.xml @@ -190,6 +190,10 @@ test/clj false + true + + none + From 93ca65457201948d6be1f994f5a046ea0fe477a1 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Thu, 3 Apr 2014 21:18:08 +0100 Subject: [PATCH 315/349] Update README.md added yourkit acknowledgement --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 13b5b2c29b..874db0112d 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ storm-kafka-0.8-plus Port of storm-kafka to support kafka >= 0.8 -##Usage +##Usage: For information on how to use this library in your project see: [https://clojars.org/net.wurstmeister.storm/storm-kafka-0.8-plus](https://clojars.org/net.wurstmeister.storm/storm-kafka-0.8-plus) @@ -12,3 +12,11 @@ For information on how to use this library in your project see: ##Example Topologies: [https://github.com/wurstmeister/storm-kafka-0.8-plus-test](https://github.com/wurstmeister/storm-kafka-0.8-plus-test) + +##Acknowledgement: + +YourKit is kindly supporting this open source project with its full-featured Java Profiler. +YourKit, LLC is the creator of innovative and intelligent tools for profiling +Java and .NET applications. Take a look at YourKit's leading software products: +YourKit Java Profiler and +YourKit .NET Profiler. From 0d3b833d86687857ec8cbfb1e01041ffc8120798 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 3 Apr 2014 14:17:14 -0700 Subject: [PATCH 316/349] STORM-273. Error while running storm topologies on Windows using "storm jar" --- storm-core/src/clj/backtype/storm/util.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storm-core/src/clj/backtype/storm/util.clj b/storm-core/src/clj/backtype/storm/util.clj index 580e9500c3..8bfacd3dc5 100644 --- a/storm-core/src/clj/backtype/storm/util.clj +++ b/storm-core/src/clj/backtype/storm/util.clj @@ -759,7 +759,7 @@ (defn zip-contains-dir? [zipfile target] (let [entries (->> zipfile (ZipFile.) .entries enumeration-seq (map (memfn getName)))] - (some? #(.startsWith % (str target file-path-separator)) entries) + (some? #(.startsWith % (str target "/")) entries) )) (defn url-encode [s] From b30de82f88696c1fb6a8cc497ed3a6a65a1b023c Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Fri, 4 Apr 2014 15:30:12 +0000 Subject: [PATCH 317/349] Add STORM-263 to changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0f8dd3d1e..0d46663977 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.9.2-incubating (unreleased) + * STORM-263: Update Kryo version to 2.21+ * STORM-187: Fix Netty error "java.lang.IllegalArgumentException: timeout value is negative" * STORM-186: fix float secs to millis long convertion * STORM-70: Upgrade to ZK-3.4.5 and curator-1.3.3 From 98cfe93010ff069af2f7c607b11e6735060e2405 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sat, 5 Apr 2014 13:19:46 +0100 Subject: [PATCH 318/349] close resources in tests --- src/test/storm/kafka/DynamicBrokersReaderTest.java | 2 ++ src/test/storm/kafka/KafkaTestBroker.java | 1 + src/test/storm/kafka/KafkaUtilsTest.java | 1 + src/test/storm/kafka/ZkCoordinatorTest.java | 4 +++- src/test/storm/kafka/bolt/KafkaBoltTest.java | 1 + 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/test/storm/kafka/DynamicBrokersReaderTest.java b/src/test/storm/kafka/DynamicBrokersReaderTest.java index 47387e35eb..d03bab30d8 100644 --- a/src/test/storm/kafka/DynamicBrokersReaderTest.java +++ b/src/test/storm/kafka/DynamicBrokersReaderTest.java @@ -43,6 +43,8 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { + dynamicBrokersReader.close(); + zookeeper.close(); server.close(); } diff --git a/src/test/storm/kafka/KafkaTestBroker.java b/src/test/storm/kafka/KafkaTestBroker.java index 7019c86403..d2a44a4c53 100644 --- a/src/test/storm/kafka/KafkaTestBroker.java +++ b/src/test/storm/kafka/KafkaTestBroker.java @@ -48,5 +48,6 @@ public int getPort() { public void shutdown() { kafka.shutdown(); + server.stop(); } } diff --git a/src/test/storm/kafka/KafkaUtilsTest.java b/src/test/storm/kafka/KafkaUtilsTest.java index a4e7f52f1a..25fa618fd2 100644 --- a/src/test/storm/kafka/KafkaUtilsTest.java +++ b/src/test/storm/kafka/KafkaUtilsTest.java @@ -42,6 +42,7 @@ public void setup() { @After public void shutdown() { + simpleConsumer.close(); broker.shutdown(); } diff --git a/src/test/storm/kafka/ZkCoordinatorTest.java b/src/test/storm/kafka/ZkCoordinatorTest.java index 35b3b4b3f1..c08ce82531 100644 --- a/src/test/storm/kafka/ZkCoordinatorTest.java +++ b/src/test/storm/kafka/ZkCoordinatorTest.java @@ -31,6 +31,7 @@ public class ZkCoordinatorTest { private Map stormConf = new HashMap(); private SpoutConfig spoutConfig; private ZkState state; + private SimpleConsumer simpleConsumer; @Before public void setUp() throws Exception { @@ -42,7 +43,7 @@ public void setUp() throws Exception { spoutConfig = new SpoutConfig(hosts, "topic", "/test", "id"); Map conf = buildZookeeperConfig(server); state = new ZkState(conf); - SimpleConsumer simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient"); + simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient"); when(dynamicPartitionConnections.register(any(Broker.class), anyInt())).thenReturn(simpleConsumer); } @@ -58,6 +59,7 @@ private Map buildZookeeperConfig(TestingServer server) { @After public void shutdown() throws Exception { + simpleConsumer.close(); broker.shutdown(); server.stop(); } diff --git a/src/test/storm/kafka/bolt/KafkaBoltTest.java b/src/test/storm/kafka/bolt/KafkaBoltTest.java index 129b0f6b68..fa5a104a73 100644 --- a/src/test/storm/kafka/bolt/KafkaBoltTest.java +++ b/src/test/storm/kafka/bolt/KafkaBoltTest.java @@ -53,6 +53,7 @@ public void initMocks() { @After public void shutdown() { + simpleConsumer.close(); broker.shutdown(); } From 9370c5ccaab437f2aa1c0f5fad55802aaa3b2b96 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sat, 5 Apr 2014 13:44:35 +0100 Subject: [PATCH 319/349] updated log messages --- src/jvm/storm/kafka/KafkaUtils.java | 6 +++++- src/jvm/storm/kafka/PartitionManager.java | 21 +++++++++------------ src/jvm/storm/kafka/ZkCoordinator.java | 14 ++++++-------- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/jvm/storm/kafka/KafkaUtils.java b/src/jvm/storm/kafka/KafkaUtils.java index 4e8b3a3b1d..0e7f601f88 100644 --- a/src/jvm/storm/kafka/KafkaUtils.java +++ b/src/jvm/storm/kafka/KafkaUtils.java @@ -204,11 +204,15 @@ public static List calculatePartitionsForTask(GlobalPartitionInformat } private static void logPartitionMapping(int totalTasks, int taskIndex, List taskPartitions) { - String taskPrefix = "[" + taskIndex + "/" + totalTasks + "] --> "; + String taskPrefix = taskId(taskIndex, totalTasks); if (taskPartitions.isEmpty()) { LOG.warn(taskPrefix + "no partitions assigned"); } else { LOG.info(taskPrefix + "assigned " + taskPartitions); } } + + public static String taskId(int taskIndex, int totalTasks) { + return "Task [" + (taskIndex + 1) + "/" + totalTasks + "] "; + } } diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index fc9d817189..915f0f9f92 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -6,7 +6,6 @@ import backtype.storm.metric.api.MeanReducer; import backtype.storm.metric.api.ReducedMetric; import backtype.storm.spout.SpoutOutputCollector; -import backtype.storm.utils.Utils; import com.google.common.collect.ImmutableMap; import kafka.api.OffsetRequest; import kafka.javaapi.consumer.SimpleConsumer; @@ -64,7 +63,7 @@ public PartitionManager(DynamicPartitionConnections connections, String topology String path = committedPath(); try { Map json = _state.readJSON(path); - LOG.info("Read partition information from: " + path + " --> " + json ); + LOG.info("Read partition information from: " + path + " --> " + json ); if (json != null) { jsonTopologyId = (String) ((Map) json.get("topology")).get("id"); jsonOffset = (Long) json.get("offset"); @@ -84,7 +83,7 @@ public PartitionManager(DynamicPartitionConnections connections, String topology LOG.info("Read last commit offset from zookeeper: " + _committedTo + "; old topology_id: " + jsonTopologyId + " - new topology_id: " + topologyInstanceId ); } - LOG.info("Starting Kafka " + _consumer.host() + ":" + id.partition + " from offset " + _committedTo); + LOG.info("Starting " + _partition + " from offset " + _committedTo); _emittedToOffset = _committedTo; _fetchAPILatencyMax = new CombinedMetric(new MaxMetric()); @@ -141,7 +140,7 @@ private void fill() { _fetchAPIMessageCount.incrBy(numMessages); if (numMessages > 0) { - LOG.info("Fetched " + numMessages + " messages from Kafka: " + _consumer.host() + ":" + _partition.partition); + LOG.info("Fetched " + numMessages + " messages from: " + _partition); } for (MessageAndOffset msg : msgs) { _pending.add(_emittedToOffset); @@ -149,7 +148,7 @@ private void fill() { _emittedToOffset = msg.nextOffset(); } if (numMessages > 0) { - LOG.info("Added " + numMessages + " messages from Kafka: " + _consumer.host() + ":" + _partition.partition + " to internal buffers"); + LOG.info("Added " + numMessages + " messages from: " + _partition + " to internal buffers"); } } @@ -175,7 +174,6 @@ public void fail(Long offset) { } public void commit() { - LOG.info("Committing offset for " + _partition); long committedTo; if (_pending.isEmpty()) { committedTo = _emittedToOffset; @@ -183,9 +181,8 @@ public void commit() { committedTo = _pending.first(); } if (committedTo != _committedTo) { - LOG.info("Writing committed offset to ZK: " + committedTo); - - Map data = (Map) ImmutableMap.builder() + LOG.info("Writing committed offset (" + committedTo + ") to ZK for " + _partition + " for topology: " + _topologyInstanceId); + Map data = ImmutableMap.builder() .put("topology", ImmutableMap.of("id", _topologyInstanceId, "name", _stormConf.get(Config.TOPOLOGY_NAME))) .put("offset", committedTo) @@ -194,11 +191,11 @@ public void commit() { "port", _partition.host.port)) .put("topic", _spoutConfig.topic).build(); _state.writeJSON(committedPath(), data); - - LOG.info("Wrote committed offset to ZK: " + committedTo); _committedTo = committedTo; + LOG.info("Wrote committed offset (" + committedTo + ") to ZK for " + _partition + " for topology: " + _topologyInstanceId); + } else { + LOG.info("No new offset for " + _partition + " for topology: " + _topologyInstanceId); } - LOG.info("Committed offset " + committedTo + " for " + _partition + " for topology: " + _topologyInstanceId); } private String committedPath() { diff --git a/src/jvm/storm/kafka/ZkCoordinator.java b/src/jvm/storm/kafka/ZkCoordinator.java index 35d2c57ea7..ec35aedb8d 100644 --- a/src/jvm/storm/kafka/ZkCoordinator.java +++ b/src/jvm/storm/kafka/ZkCoordinator.java @@ -6,6 +6,8 @@ import java.util.*; +import static storm.kafka.KafkaUtils.taskId; + public class ZkCoordinator implements PartitionCoordinator { public static final Logger LOG = LoggerFactory.getLogger(ZkCoordinator.class); @@ -55,7 +57,7 @@ public List getMyManagedPartitions() { void refresh() { try { - LOG.info(taskIdentifier() + "Refreshing partition manager connections"); + LOG.info(taskId(_taskIndex, _totalTasks) + "Refreshing partition manager connections"); GlobalPartitionInformation brokerInfo = _reader.getBrokerInfo(); List mine = KafkaUtils.calculatePartitionsForTask(brokerInfo, _totalTasks, _taskIndex); @@ -66,13 +68,13 @@ void refresh() { Set deletedPartitions = new HashSet(curr); deletedPartitions.removeAll(mine); - LOG.info(taskIdentifier() + "Deleted partition managers: " + deletedPartitions.toString()); + LOG.info(taskId(_taskIndex, _totalTasks) + "Deleted partition managers: " + deletedPartitions.toString()); for (Partition id : deletedPartitions) { PartitionManager man = _managers.remove(id); man.close(); } - LOG.info(taskIdentifier() + "New partition managers: " + newPartitions.toString()); + LOG.info(taskId(_taskIndex, _totalTasks) + "New partition managers: " + newPartitions.toString()); for (Partition id : newPartitions) { PartitionManager man = new PartitionManager(_connections, _topologyInstanceId, _state, _stormConf, _spoutConfig, id); @@ -83,11 +85,7 @@ void refresh() { throw new RuntimeException(e); } _cachedList = new ArrayList(_managers.values()); - LOG.info(taskIdentifier() + "Finished refreshing"); - } - - private String taskIdentifier() { - return "[" + _taskIndex + "/" + _totalTasks + "] - "; + LOG.info(taskId(_taskIndex, _totalTasks) + "Finished refreshing"); } @Override From c8c04a6a6d9575f700d4c3db35927ddb5347a265 Mon Sep 17 00:00:00 2001 From: wurstmeister Date: Sat, 5 Apr 2014 14:03:49 +0100 Subject: [PATCH 320/349] removed duplicated code --- src/jvm/storm/kafka/PartitionManager.java | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/jvm/storm/kafka/PartitionManager.java b/src/jvm/storm/kafka/PartitionManager.java index 915f0f9f92..03075bbb4b 100644 --- a/src/jvm/storm/kafka/PartitionManager.java +++ b/src/jvm/storm/kafka/PartitionManager.java @@ -174,25 +174,20 @@ public void fail(Long offset) { } public void commit() { - long committedTo; - if (_pending.isEmpty()) { - committedTo = _emittedToOffset; - } else { - committedTo = _pending.first(); - } - if (committedTo != _committedTo) { - LOG.info("Writing committed offset (" + committedTo + ") to ZK for " + _partition + " for topology: " + _topologyInstanceId); + long lastCompletedOffset = lastCompletedOffset(); + if (lastCompletedOffset != lastCommittedOffset()) { + LOG.info("Writing last completed offset (" + lastCompletedOffset + ") to ZK for " + _partition + " for topology: " + _topologyInstanceId); Map data = ImmutableMap.builder() .put("topology", ImmutableMap.of("id", _topologyInstanceId, "name", _stormConf.get(Config.TOPOLOGY_NAME))) - .put("offset", committedTo) + .put("offset", lastCompletedOffset) .put("partition", _partition.partition) .put("broker", ImmutableMap.of("host", _partition.host.host, "port", _partition.host.port)) .put("topic", _spoutConfig.topic).build(); _state.writeJSON(committedPath(), data); - _committedTo = committedTo; - LOG.info("Wrote committed offset (" + committedTo + ") to ZK for " + _partition + " for topology: " + _topologyInstanceId); + _committedTo = lastCompletedOffset; + LOG.info("Wrote last completed offset (" + lastCompletedOffset + ") to ZK for " + _partition + " for topology: " + _topologyInstanceId); } else { LOG.info("No new offset for " + _partition + " for topology: " + _topologyInstanceId); } From 2c8d3c8927ae611b2c8c7000f8da85df22326b7b Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Mon, 7 Apr 2014 15:15:13 +0000 Subject: [PATCH 321/349] STROM-247: Replace links to github resources in storm script (for Julien Nioche) --- CHANGELOG.md | 1 + bin/storm | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d46663977..8b2f2d12ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.9.2-incubating (unreleased) + * STROM-247: Replace links to github resources in storm script * STORM-263: Update Kryo version to 2.21+ * STORM-187: Fix Netty error "java.lang.IllegalArgumentException: timeout value is negative" * STORM-186: fix float secs to millis long convertion diff --git a/bin/storm b/bin/storm index aee46cbfa3..c85ae8ebe7 100755 --- a/bin/storm +++ b/bin/storm @@ -147,7 +147,7 @@ def jar(jarfile, klass, *args): Runs the main method of class with the specified arguments. The storm jars and configs in ~/.storm are put on the classpath. The process is configured so that StormSubmitter - (http://nathanmarz.github.com/storm/doc/backtype/storm/StormSubmitter.html) + (http://storm.incubator.apache.org/apidocs/backtype/storm/StormSubmitter.html) will upload the jar at topology-jar-path when the topology is submitted. """ exec_storm_class( @@ -262,7 +262,7 @@ def nimbus(klass="backtype.storm.daemon.nimbus"): supervision with a tool like daemontools or monit. See Setting up a Storm cluster for more information. - (https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster) + (http://storm.incubator.apache.org/documentation/Setting-up-a-Storm-cluster) """ cppaths = [CLUSTER_CONF_DIR] jvmopts = parse_args(confvalue("nimbus.childopts", cppaths)) + [ @@ -282,7 +282,7 @@ def supervisor(klass="backtype.storm.daemon.supervisor"): under supervision with a tool like daemontools or monit. See Setting up a Storm cluster for more information. - (https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster) + (http://storm.incubator.apache.org/documentation/Setting-up-a-Storm-cluster) """ cppaths = [CLUSTER_CONF_DIR] jvmopts = parse_args(confvalue("supervisor.childopts", cppaths)) + [ @@ -303,7 +303,7 @@ def ui(): should be run under supervision with a tool like daemontools or monit. See Setting up a Storm cluster for more information. - (https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster) + (http://storm.incubator.apache.org/documentation/Setting-up-a-Storm-cluster) """ cppaths = [CLUSTER_CONF_DIR] jvmopts = parse_args(confvalue("ui.childopts", cppaths)) + [ @@ -324,7 +324,7 @@ def logviewer(): tool like daemontools or monit. See Setting up a Storm cluster for more information. - (https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster) + (http://storm.incubator.apache.org/documentation/Setting-up-a-Storm-cluster) """ cppaths = [CLUSTER_CONF_DIR] jvmopts = parse_args(confvalue("logviewer.childopts", cppaths)) + [ @@ -344,7 +344,7 @@ def drpc(): with a tool like daemontools or monit. See Distributed RPC for more information. - (https://github.com/nathanmarz/storm/wiki/Distributed-RPC) + (http://storm.incubator.apache.org/documentation/Distributed-RPC) """ cppaths = [CLUSTER_CONF_DIR] jvmopts = parse_args(confvalue("drpc.childopts", cppaths)) + [ @@ -392,7 +392,7 @@ def print_commands(): """Print all client commands and link to documentation""" print "Commands:\n\t", "\n\t".join(sorted(COMMANDS.keys())) print "\nHelp:", "\n\thelp", "\n\thelp " - print "\nDocumentation for the storm client can be found at https://github.com/nathanmarz/storm/wiki/Command-line-client\n" + print "\nDocumentation for the storm client can be found at http://storm.incubator.apache.org/documentation/Command-line-client.html\n" print "Configs can be overridden using one or more -c flags, e.g. \"storm list -c nimbus.host=nimbus.mycompany.com\"\n" def print_usage(command=None): From 38ea0ca63651dcf535c4ac0b5bf4e48246e5236f Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Tue, 8 Apr 2014 16:08:06 +0000 Subject: [PATCH 322/349] STORM-273 updated change log. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b2f2d12ea..91fa43b5a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.9.2-incubating (unreleased) + * STORM-273: Error while running storm topologies on Windows using "storm jar" * STROM-247: Replace links to github resources in storm script * STORM-263: Update Kryo version to 2.21+ * STORM-187: Fix Netty error "java.lang.IllegalArgumentException: timeout value is negative" From bde4f880dc69ad481531a8ed9670e102640f128b Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Wed, 9 Apr 2014 14:30:32 +0000 Subject: [PATCH 323/349] Added STORM-270 to changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91fa43b5a2..ae9b86e25d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.9.2-incubating (unreleased) + * STORM-270: don't package .clj files in release jars. * STORM-273: Error while running storm topologies on Windows using "storm jar" * STROM-247: Replace links to github resources in storm script * STORM-263: Update Kryo version to 2.21+ @@ -6,6 +7,7 @@ * STORM-186: fix float secs to millis long convertion * STORM-70: Upgrade to ZK-3.4.5 and curator-1.3.3 * STORM-146: Unit test regression when storm is compiled with 3.4.5 zookeeper + ## 0.9.1-incubating * Fix to prevent Nimbus from hanging if random data is sent to nimbus thrift port * Improved support for running on Windows platforms From a72aafae503c8314b26330e0d4f440fb4632a483 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 9 Apr 2014 10:33:16 -0400 Subject: [PATCH 324/349] update to kafka 0.8.1 --- external/storm-kafka/pom.xml | 2 +- .../storm-kafka/src/test/storm/kafka/KafkaTestBroker.java | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/external/storm-kafka/pom.xml b/external/storm-kafka/pom.xml index 15743b6ac3..938567814d 100644 --- a/external/storm-kafka/pom.xml +++ b/external/storm-kafka/pom.xml @@ -106,7 +106,7 @@ org.apache.kafka ${kafkaArtifact} - 0.8.0 + 0.8.1 org.apache.zookeeper diff --git a/external/storm-kafka/src/test/storm/kafka/KafkaTestBroker.java b/external/storm-kafka/src/test/storm/kafka/KafkaTestBroker.java index d2a44a4c53..db4663d279 100644 --- a/external/storm-kafka/src/test/storm/kafka/KafkaTestBroker.java +++ b/external/storm-kafka/src/test/storm/kafka/KafkaTestBroker.java @@ -6,6 +6,7 @@ import com.netflix.curator.test.TestingServer; import kafka.server.KafkaServerStartable; +import java.io.IOException; import java.util.Properties; /** @@ -48,6 +49,10 @@ public int getPort() { public void shutdown() { kafka.shutdown(); - server.stop(); + try { + server.stop(); + } catch (IOException e) { + e.printStackTrace(); + } } } From 859a2e81be8f61720fc2965acef5f73f1449a7bb Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 9 Apr 2014 10:57:37 -0400 Subject: [PATCH 325/349] add topic to committed path --- .../jvm/storm/kafka/DynamicBrokersReader.java | 18 +++++++++++------- .../src/jvm/storm/kafka/KafkaUtils.java | 4 ++-- .../src/jvm/storm/kafka/PartitionManager.java | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/external/storm-kafka/src/jvm/storm/kafka/DynamicBrokersReader.java b/external/storm-kafka/src/jvm/storm/kafka/DynamicBrokersReader.java index cd751fefb2..b9085bcf33 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/DynamicBrokersReader.java +++ b/external/storm-kafka/src/jvm/storm/kafka/DynamicBrokersReader.java @@ -26,13 +26,17 @@ public class DynamicBrokersReader { public DynamicBrokersReader(Map conf, String zkStr, String zkPath, String topic) { _zkPath = zkPath; _topic = topic; - _curator = CuratorFrameworkFactory.newClient( - zkStr, - Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), - 15000, - new RetryNTimes(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), - Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); - _curator.start(); + try { + _curator = CuratorFrameworkFactory.newClient( + zkStr, + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), + 15000, + new RetryNTimes(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), + Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); + _curator.start(); + } catch (Exception ex) { + LOG.error("can't connect to zookeeper"); + } } /** diff --git a/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java b/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java index 0e7f601f88..313e08ed95 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java +++ b/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java @@ -41,7 +41,7 @@ public static IBrokerReader makeBrokerReader(Map stormConf, KafkaConfig conf) { public static long getOffset(SimpleConsumer consumer, String topic, int partition, KafkaConfig config) { long startOffsetTime = kafka.api.OffsetRequest.LatestTime(); - if (config.forceFromStart) { + if ( config.forceFromStart ) { startOffsetTime = config.startOffsetTime; } return getOffset(consumer, topic, partition, startOffsetTime); @@ -93,7 +93,7 @@ public Object getValueAndReset() { LOG.warn("partitionToOffset contains partition not found in _connections. Stale partition data?"); return null; } - long earliestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.EarliestTime()); + long earliestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.EarliestTime()); long latestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.LatestTime()); if (earliestTimeOffset == 0 || latestTimeOffset == 0) { LOG.warn("No data found in Kafka Partition " + partition.getId()); diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java index 03075bbb4b..68151eb6e0 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java +++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java @@ -194,7 +194,7 @@ public void commit() { } private String committedPath() { - return _spoutConfig.zkRoot + "/" + _spoutConfig.id + "/" + _partition.getId(); + return _spoutConfig.zkRoot + "/" + _spoutConfig.id + "/" + _spoutConfig.topic + "/" + _partition.getId(); } public long queryPartitionOffsetLatestTime() { From 3270dd2a83f5b78d51f23ae1b8b6e3951c60b2c0 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 9 Apr 2014 11:26:33 -0400 Subject: [PATCH 326/349] update README --- external/storm-kafka/README.md | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/external/storm-kafka/README.md b/external/storm-kafka/README.md index 874db0112d..726cff648e 100644 --- a/external/storm-kafka/README.md +++ b/external/storm-kafka/README.md @@ -1,22 +1,25 @@ -storm-kafka-0.8-plus +Storm Kafka ==================== -Port of storm-kafka to support kafka >= 0.8 +Provides core storm and Trident spout implementations for consuming data from Apache Kafka 0.8.x. -##Usage: -For information on how to use this library in your project see: -[https://clojars.org/net.wurstmeister.storm/storm-kafka-0.8-plus](https://clojars.org/net.wurstmeister.storm/storm-kafka-0.8-plus) +##Usage Example -##Example Topologies: +```java + + TridentTopology topology = new TridentTopology(); -[https://github.com/wurstmeister/storm-kafka-0.8-plus-test](https://github.com/wurstmeister/storm-kafka-0.8-plus-test) + BrokerHosts zk = new ZkHosts("localhost"); -##Acknowledgement: + TridentKafkaConfig spoutConf = new TridentKafkaConfig(zk, "test-topic"); + spoutConf.scheme = new SchemeAsMultiScheme(new StringScheme()); + OpaqueTridentKafkaSpout spout = new OpaqueTridentKafkaSpout(spoutConf); -YourKit is kindly supporting this open source project with its full-featured Java Profiler. -YourKit, LLC is the creator of innovative and intelligent tools for profiling -Java and .NET applications. Take a look at YourKit's leading software products: -YourKit Java Profiler and -YourKit .NET Profiler. + +``` + +## Committer Sponsors + + * P. Taylor Goetz ([ptgoetz@apache.org](mailto:ptgoetz@apache.org)) \ No newline at end of file From 72dbbee75fe2524ac4ffc2da6f5166483af7ad32 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 9 Apr 2014 11:40:53 -0400 Subject: [PATCH 327/349] apply Apache license headers --- external/storm-kafka/CHANGELOG.md | 5 +- external/storm-kafka/LICENSE | 191 ------------------ .../src/jvm/storm/kafka/Broker.java | 17 ++ .../src/jvm/storm/kafka/BrokerHosts.java | 22 +- .../jvm/storm/kafka/DynamicBrokersReader.java | 17 ++ .../kafka/DynamicPartitionConnections.java | 17 ++ .../jvm/storm/kafka/FailedFetchException.java | 17 ++ .../src/jvm/storm/kafka/KafkaConfig.java | 17 ++ .../src/jvm/storm/kafka/KafkaError.java | 21 +- .../src/jvm/storm/kafka/KafkaSpout.java | 17 ++ .../src/jvm/storm/kafka/KafkaUtils.java | 17 ++ .../src/jvm/storm/kafka/KeyValueScheme.java | 17 ++ .../kafka/KeyValueSchemeAsMultiScheme.java | 17 ++ .../src/jvm/storm/kafka/Partition.java | 17 ++ .../jvm/storm/kafka/PartitionCoordinator.java | 17 ++ .../src/jvm/storm/kafka/PartitionManager.java | 17 ++ .../src/jvm/storm/kafka/SpoutConfig.java | 17 ++ .../jvm/storm/kafka/StaticCoordinator.java | 17 ++ .../src/jvm/storm/kafka/StaticHosts.java | 17 ++ .../kafka/StaticPartitionConnections.java | 17 ++ .../jvm/storm/kafka/StringKeyValueScheme.java | 17 ++ .../src/jvm/storm/kafka/StringScheme.java | 17 ++ .../src/jvm/storm/kafka/ZkCoordinator.java | 17 ++ .../src/jvm/storm/kafka/ZkHosts.java | 22 +- .../src/jvm/storm/kafka/ZkState.java | 17 ++ .../src/jvm/storm/kafka/bolt/KafkaBolt.java | 17 ++ .../jvm/storm/kafka/trident/Coordinator.java | 21 +- .../kafka/trident/DefaultCoordinator.java | 17 ++ .../trident/GlobalPartitionInformation.java | 22 +- .../kafka/trident/IBatchCoordinator.java | 17 ++ .../storm/kafka/trident/IBrokerReader.java | 17 ++ .../jvm/storm/kafka/trident/MaxMetric.java | 17 ++ .../trident/OpaqueTridentKafkaSpout.java | 17 ++ .../kafka/trident/StaticBrokerReader.java | 17 ++ .../TransactionalTridentKafkaSpout.java | 17 ++ .../kafka/trident/TridentKafkaConfig.java | 17 ++ .../kafka/trident/TridentKafkaEmitter.java | 21 +- .../storm/kafka/trident/ZkBrokerReader.java | 17 ++ 38 files changed, 619 insertions(+), 216 deletions(-) delete mode 100644 external/storm-kafka/LICENSE diff --git a/external/storm-kafka/CHANGELOG.md b/external/storm-kafka/CHANGELOG.md index f87642148e..33a49eeccd 100644 --- a/external/storm-kafka/CHANGELOG.md +++ b/external/storm-kafka/CHANGELOG.md @@ -1,9 +1,12 @@ -## 0.5.0 +## 0.9.2-incubating (0.5.0) +* incorporated as an Apache Storm external module * fixed partition assignment for KafkaSpout * upgraded to storm 0.9.1 + ## 0.4.0 * added support for reading kafka message keys * configurable metrics emit interval + ## 0.3.0 * updated partition path in zookeeper * added error handling for fetch request diff --git a/external/storm-kafka/LICENSE b/external/storm-kafka/LICENSE deleted file mode 100644 index 37ec93a14f..0000000000 --- a/external/storm-kafka/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising -permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. - -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that -is based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work -by the copyright owner or by an individual or Legal Entity authorized to submit -on behalf of the copyright owner. For the purposes of this definition, -"submitted" means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, and -issue tracking systems that are managed by, or on behalf of, the Licensor for -the purpose of discussing and improving the Work, but excluding communication -that is conspicuously marked or otherwise designated in writing by the copyright -owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf -of whom a Contribution has been received by Licensor and subsequently -incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the Work and such -Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable (except as stated in this section) patent license to make, have -made, use, offer to sell, sell, import, and otherwise transfer the Work, where -such license applies only to those patent claims licensable by such Contributor -that are necessarily infringed by their Contribution(s) alone or by combination -of their Contribution(s) with the Work to which such Contribution(s) was -submitted. If You institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work or a -Contribution incorporated within the Work constitutes direct or contributory -patent infringement, then any patent licenses granted to You under this License -for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof -in any medium, with or without modifications, and in Source or Object form, -provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of -this License; and -You must cause any modified files to carry prominent notices stating that You -changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, -all copyright, patent, trademark, and attribution notices from the Source form -of the Work, excluding those notices that do not pertain to any part of the -Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any -Derivative Works that You distribute must include a readable copy of the -attribution notices contained within such NOTICE file, excluding those notices -that do not pertain to any part of the Derivative Works, in at least one of the -following places: within a NOTICE text file distributed as part of the -Derivative Works; within the Source form or documentation, if provided along -with the Derivative Works; or, within a display generated by the Derivative -Works, if and wherever such third-party notices normally appear. The contents of -the NOTICE file are for informational purposes only and do not modify the -License. You may add Your own attribution notices within Derivative Works that -You distribute, alongside or as an addendum to the NOTICE text from the Work, -provided that such additional attribution notices cannot be construed as -modifying the License. -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted -for inclusion in the Work by You to the Licensor shall be under the terms and -conditions of this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify the terms of -any separate license agreement you may have executed with Licensor regarding -such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, -service marks, or product names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the -Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, -including, without limitation, any warranties or conditions of TITLE, -NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are -solely responsible for determining the appropriateness of using or -redistributing the Work and assume any risks associated with Your exercise of -permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), -contract, or otherwise, unless required by applicable law (such as deliberate -and grossly negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, incidental, -or consequential damages of any character arising as a result of this License or -out of the use or inability to use the Work (including but not limited to -damages for loss of goodwill, work stoppage, computer failure or malfunction, or -any and all other commercial damages or losses), even if such Contributor has -been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to -offer, and charge a fee for, acceptance of support, warranty, indemnity, or -other liability obligations and/or rights consistent with this License. However, -in accepting such obligations, You may act only on Your own behalf and on Your -sole responsibility, not on behalf of any other Contributor, and only if You -agree to indemnify, defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason of your -accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work - -To apply the Apache License to your work, attach the following boilerplate -notice, with the fields enclosed by brackets "[]" replaced with your own -identifying information. (Don't include the brackets!) The text should be -enclosed in the appropriate comment syntax for the file format. We also -recommend that a file or class name and description of purpose be included on -the same "printed page" as the copyright notice for easier identification within -third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/external/storm-kafka/src/jvm/storm/kafka/Broker.java b/external/storm-kafka/src/jvm/storm/kafka/Broker.java index 2451eee454..bfa3e0b0b6 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/Broker.java +++ b/external/storm-kafka/src/jvm/storm/kafka/Broker.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import java.io.Serializable; diff --git a/external/storm-kafka/src/jvm/storm/kafka/BrokerHosts.java b/external/storm-kafka/src/jvm/storm/kafka/BrokerHosts.java index 12ef7b1168..fcdf0b6140 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/BrokerHosts.java +++ b/external/storm-kafka/src/jvm/storm/kafka/BrokerHosts.java @@ -1,11 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import java.io.Serializable; -/** - * Date: 11/05/2013 - * Time: 14:40 - */ + public interface BrokerHosts extends Serializable { } \ No newline at end of file diff --git a/external/storm-kafka/src/jvm/storm/kafka/DynamicBrokersReader.java b/external/storm-kafka/src/jvm/storm/kafka/DynamicBrokersReader.java index b9085bcf33..51978623fc 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/DynamicBrokersReader.java +++ b/external/storm-kafka/src/jvm/storm/kafka/DynamicBrokersReader.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import backtype.storm.Config; diff --git a/external/storm-kafka/src/jvm/storm/kafka/DynamicPartitionConnections.java b/external/storm-kafka/src/jvm/storm/kafka/DynamicPartitionConnections.java index 8d0115b68d..f550858342 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/DynamicPartitionConnections.java +++ b/external/storm-kafka/src/jvm/storm/kafka/DynamicPartitionConnections.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import kafka.javaapi.consumer.SimpleConsumer; diff --git a/external/storm-kafka/src/jvm/storm/kafka/FailedFetchException.java b/external/storm-kafka/src/jvm/storm/kafka/FailedFetchException.java index 0bd1123ac7..011240e4ce 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/FailedFetchException.java +++ b/external/storm-kafka/src/jvm/storm/kafka/FailedFetchException.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; public class FailedFetchException extends RuntimeException { diff --git a/external/storm-kafka/src/jvm/storm/kafka/KafkaConfig.java b/external/storm-kafka/src/jvm/storm/kafka/KafkaConfig.java index 8ef2a88b7a..875629b49b 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/KafkaConfig.java +++ b/external/storm-kafka/src/jvm/storm/kafka/KafkaConfig.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import backtype.storm.spout.MultiScheme; diff --git a/external/storm-kafka/src/jvm/storm/kafka/KafkaError.java b/external/storm-kafka/src/jvm/storm/kafka/KafkaError.java index a67335c3c1..634af85510 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/KafkaError.java +++ b/external/storm-kafka/src/jvm/storm/kafka/KafkaError.java @@ -1,9 +1,22 @@ -package storm.kafka; - /** - * Date: 11/01/2014 - * Time: 14:21 + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ +package storm.kafka; + public enum KafkaError { NO_ERROR, OFFSET_OUT_OF_RANGE, diff --git a/external/storm-kafka/src/jvm/storm/kafka/KafkaSpout.java b/external/storm-kafka/src/jvm/storm/kafka/KafkaSpout.java index 79e33fe833..102dce1936 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/KafkaSpout.java +++ b/external/storm-kafka/src/jvm/storm/kafka/KafkaSpout.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import backtype.storm.Config; diff --git a/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java b/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java index 313e08ed95..3d355e56a7 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java +++ b/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import backtype.storm.metric.api.IMetric; diff --git a/external/storm-kafka/src/jvm/storm/kafka/KeyValueScheme.java b/external/storm-kafka/src/jvm/storm/kafka/KeyValueScheme.java index df31cb8b3b..f42f7c82d3 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/KeyValueScheme.java +++ b/external/storm-kafka/src/jvm/storm/kafka/KeyValueScheme.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import backtype.storm.spout.Scheme; diff --git a/external/storm-kafka/src/jvm/storm/kafka/KeyValueSchemeAsMultiScheme.java b/external/storm-kafka/src/jvm/storm/kafka/KeyValueSchemeAsMultiScheme.java index 2412a1ce78..a570e7db80 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/KeyValueSchemeAsMultiScheme.java +++ b/external/storm-kafka/src/jvm/storm/kafka/KeyValueSchemeAsMultiScheme.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import backtype.storm.spout.SchemeAsMultiScheme; diff --git a/external/storm-kafka/src/jvm/storm/kafka/Partition.java b/external/storm-kafka/src/jvm/storm/kafka/Partition.java index 96a3ad7350..fe6ab4f963 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/Partition.java +++ b/external/storm-kafka/src/jvm/storm/kafka/Partition.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import com.google.common.base.Objects; diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionCoordinator.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionCoordinator.java index d28248d1ec..60cc237bd8 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/PartitionCoordinator.java +++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionCoordinator.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import java.util.List; diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java index 68151eb6e0..9504427108 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java +++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import backtype.storm.Config; diff --git a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java index 05551ece2b..1b66026e1e 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java +++ b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import java.io.Serializable; diff --git a/external/storm-kafka/src/jvm/storm/kafka/StaticCoordinator.java b/external/storm-kafka/src/jvm/storm/kafka/StaticCoordinator.java index 040060c524..456a2a1344 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/StaticCoordinator.java +++ b/external/storm-kafka/src/jvm/storm/kafka/StaticCoordinator.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import java.util.ArrayList; diff --git a/external/storm-kafka/src/jvm/storm/kafka/StaticHosts.java b/external/storm-kafka/src/jvm/storm/kafka/StaticHosts.java index 9ed719388f..bee71187fd 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/StaticHosts.java +++ b/external/storm-kafka/src/jvm/storm/kafka/StaticHosts.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import storm.kafka.trident.GlobalPartitionInformation; diff --git a/external/storm-kafka/src/jvm/storm/kafka/StaticPartitionConnections.java b/external/storm-kafka/src/jvm/storm/kafka/StaticPartitionConnections.java index a9b9db1100..1353b6c055 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/StaticPartitionConnections.java +++ b/external/storm-kafka/src/jvm/storm/kafka/StaticPartitionConnections.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import kafka.javaapi.consumer.SimpleConsumer; diff --git a/external/storm-kafka/src/jvm/storm/kafka/StringKeyValueScheme.java b/external/storm-kafka/src/jvm/storm/kafka/StringKeyValueScheme.java index a6adddb95c..41cacb6149 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/StringKeyValueScheme.java +++ b/external/storm-kafka/src/jvm/storm/kafka/StringKeyValueScheme.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import backtype.storm.tuple.Values; diff --git a/external/storm-kafka/src/jvm/storm/kafka/StringScheme.java b/external/storm-kafka/src/jvm/storm/kafka/StringScheme.java index a8094484ef..102ea69583 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/StringScheme.java +++ b/external/storm-kafka/src/jvm/storm/kafka/StringScheme.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import backtype.storm.spout.Scheme; diff --git a/external/storm-kafka/src/jvm/storm/kafka/ZkCoordinator.java b/external/storm-kafka/src/jvm/storm/kafka/ZkCoordinator.java index ec35aedb8d..e414d063ae 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/ZkCoordinator.java +++ b/external/storm-kafka/src/jvm/storm/kafka/ZkCoordinator.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import org.slf4j.Logger; diff --git a/external/storm-kafka/src/jvm/storm/kafka/ZkHosts.java b/external/storm-kafka/src/jvm/storm/kafka/ZkHosts.java index f2e0fc24b3..d9acc66267 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/ZkHosts.java +++ b/external/storm-kafka/src/jvm/storm/kafka/ZkHosts.java @@ -1,9 +1,23 @@ -package storm.kafka; - /** - * Date: 11/05/2013 - * Time: 14:38 + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ +package storm.kafka; + + public class ZkHosts implements BrokerHosts { private static final String DEFAULT_ZK_PATH = "/brokers"; diff --git a/external/storm-kafka/src/jvm/storm/kafka/ZkState.java b/external/storm-kafka/src/jvm/storm/kafka/ZkState.java index d5416af406..52585efd17 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/ZkState.java +++ b/external/storm-kafka/src/jvm/storm/kafka/ZkState.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka; import backtype.storm.Config; diff --git a/external/storm-kafka/src/jvm/storm/kafka/bolt/KafkaBolt.java b/external/storm-kafka/src/jvm/storm/kafka/bolt/KafkaBolt.java index 89969d9a92..b9ea948599 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/bolt/KafkaBolt.java +++ b/external/storm-kafka/src/jvm/storm/kafka/bolt/KafkaBolt.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka.bolt; import backtype.storm.task.OutputCollector; diff --git a/external/storm-kafka/src/jvm/storm/kafka/trident/Coordinator.java b/external/storm-kafka/src/jvm/storm/kafka/trident/Coordinator.java index f67acaafd4..c395f8ca8f 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/trident/Coordinator.java +++ b/external/storm-kafka/src/jvm/storm/kafka/trident/Coordinator.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka.trident; import storm.kafka.KafkaUtils; @@ -6,10 +23,6 @@ import java.util.Map; -/** - * Date: 11/05/2013 - * Time: 19:35 - */ class Coordinator implements IPartitionedTridentSpout.Coordinator, IOpaquePartitionedTridentSpout.Coordinator { private IBrokerReader reader; diff --git a/external/storm-kafka/src/jvm/storm/kafka/trident/DefaultCoordinator.java b/external/storm-kafka/src/jvm/storm/kafka/trident/DefaultCoordinator.java index 89cd5034ac..04e1396b79 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/trident/DefaultCoordinator.java +++ b/external/storm-kafka/src/jvm/storm/kafka/trident/DefaultCoordinator.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka.trident; public class DefaultCoordinator implements IBatchCoordinator { diff --git a/external/storm-kafka/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java b/external/storm-kafka/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java index 6f82f62ddb..ae136e569d 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java +++ b/external/storm-kafka/src/jvm/storm/kafka/trident/GlobalPartitionInformation.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka.trident; import storm.kafka.Broker; @@ -8,10 +25,7 @@ import com.google.common.base.Objects; -/** - * Date: 14/05/2013 - * Time: 19:18 - */ + public class GlobalPartitionInformation implements Iterable, Serializable { private Map partitionMap; diff --git a/external/storm-kafka/src/jvm/storm/kafka/trident/IBatchCoordinator.java b/external/storm-kafka/src/jvm/storm/kafka/trident/IBatchCoordinator.java index 1b8a8ce337..04231f430c 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/trident/IBatchCoordinator.java +++ b/external/storm-kafka/src/jvm/storm/kafka/trident/IBatchCoordinator.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka.trident; import java.io.Serializable; diff --git a/external/storm-kafka/src/jvm/storm/kafka/trident/IBrokerReader.java b/external/storm-kafka/src/jvm/storm/kafka/trident/IBrokerReader.java index 73c9738ea8..3e018d9956 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/trident/IBrokerReader.java +++ b/external/storm-kafka/src/jvm/storm/kafka/trident/IBrokerReader.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka.trident; public interface IBrokerReader { diff --git a/external/storm-kafka/src/jvm/storm/kafka/trident/MaxMetric.java b/external/storm-kafka/src/jvm/storm/kafka/trident/MaxMetric.java index a8f88ba410..60d7c7b525 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/trident/MaxMetric.java +++ b/external/storm-kafka/src/jvm/storm/kafka/trident/MaxMetric.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka.trident; diff --git a/external/storm-kafka/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java b/external/storm-kafka/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java index 35b7033ffb..136e7d252e 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java +++ b/external/storm-kafka/src/jvm/storm/kafka/trident/OpaqueTridentKafkaSpout.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka.trident; import backtype.storm.task.TopologyContext; diff --git a/external/storm-kafka/src/jvm/storm/kafka/trident/StaticBrokerReader.java b/external/storm-kafka/src/jvm/storm/kafka/trident/StaticBrokerReader.java index 98a8f53204..d1673f1078 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/trident/StaticBrokerReader.java +++ b/external/storm-kafka/src/jvm/storm/kafka/trident/StaticBrokerReader.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka.trident; public class StaticBrokerReader implements IBrokerReader { diff --git a/external/storm-kafka/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java b/external/storm-kafka/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java index b32d301026..7a44bdd72f 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java +++ b/external/storm-kafka/src/jvm/storm/kafka/trident/TransactionalTridentKafkaSpout.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka.trident; import backtype.storm.task.TopologyContext; diff --git a/external/storm-kafka/src/jvm/storm/kafka/trident/TridentKafkaConfig.java b/external/storm-kafka/src/jvm/storm/kafka/trident/TridentKafkaConfig.java index 073afa2592..3878cc8900 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/trident/TridentKafkaConfig.java +++ b/external/storm-kafka/src/jvm/storm/kafka/trident/TridentKafkaConfig.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka.trident; import storm.kafka.BrokerHosts; diff --git a/external/storm-kafka/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java b/external/storm-kafka/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java index 973ce8f7b9..8c5768148f 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java +++ b/external/storm-kafka/src/jvm/storm/kafka/trident/TridentKafkaEmitter.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka.trident; import backtype.storm.Config; @@ -27,10 +44,6 @@ import java.util.List; import java.util.Map; -/** - * Date: 21/05/2013 - * Time: 08:38 - */ public class TridentKafkaEmitter { public static final Logger LOG = LoggerFactory.getLogger(TridentKafkaEmitter.class); diff --git a/external/storm-kafka/src/jvm/storm/kafka/trident/ZkBrokerReader.java b/external/storm-kafka/src/jvm/storm/kafka/trident/ZkBrokerReader.java index 5e2361d13c..db1132893a 100644 --- a/external/storm-kafka/src/jvm/storm/kafka/trident/ZkBrokerReader.java +++ b/external/storm-kafka/src/jvm/storm/kafka/trident/ZkBrokerReader.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package storm.kafka.trident; import org.slf4j.Logger; From eb3ecb37a3a6b5335a01a3a57b5943f6cfa51137 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 9 Apr 2014 13:47:16 -0400 Subject: [PATCH 328/349] STORM-258: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae9b86e25d..1cd7e7539b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.9.2-incubating (unreleased) + * STORM-258: Update commons-io version to 2.4 * STORM-270: don't package .clj files in release jars. * STORM-273: Error while running storm topologies on Windows using "storm jar" * STROM-247: Replace links to github resources in storm script From 89e2fbbdb187e7f8bdbd75a5710aa3c61ceb4632 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 9 Apr 2014 14:15:17 -0400 Subject: [PATCH 329/349] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cd7e7539b..a7131e0163 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.9.2-incubating (unreleased) + * Fix a potential race condition with simulated time in Storm's unit tests * STORM-258: Update commons-io version to 2.4 * STORM-270: don't package .clj files in release jars. * STORM-273: Error while running storm topologies on Windows using "storm jar" From 6f0f30c5f7147c65c534b5161804626d94596b5e Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Wed, 9 Apr 2014 14:48:08 -0400 Subject: [PATCH 330/349] add JIRA # to changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7131e0163..1e925c5832 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## 0.9.2-incubating (unreleased) - * Fix a potential race condition with simulated time in Storm's unit tests + * STORM-260: Fix a potential race condition with simulated time in Storm's unit tests * STORM-258: Update commons-io version to 2.4 * STORM-270: don't package .clj files in release jars. * STORM-273: Error while running storm topologies on Windows using "storm jar" From 2cd8f99ead495eda22e37953d140d1d3140c5430 Mon Sep 17 00:00:00 2001 From: Jason Jackson Date: Thu, 10 Apr 2014 12:51:21 -0700 Subject: [PATCH 331/349] updated my e-mail address --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 97d2b7c090..8894aaf1c4 100644 --- a/pom.xml +++ b/pom.xml @@ -114,7 +114,7 @@ jjackson Jason Jackson - jason@cvk.ca + jasonjckn@gmail.com Committer From 9da324e12ff12dec97490197063ff35f6935ca93 Mon Sep 17 00:00:00 2001 From: Derek Dagit Date: Thu, 10 Apr 2014 21:38:45 +0000 Subject: [PATCH 332/349] clean up fd after reading config --- storm-core/src/jvm/backtype/storm/utils/Utils.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java index 0e047281a3..7979e4ba15 100644 --- a/storm-core/src/jvm/backtype/storm/utils/Utils.java +++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java @@ -29,6 +29,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.FileOutputStream; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.IOException; import java.io.ObjectInputStream; @@ -135,7 +136,13 @@ public static Map findAndReadConfigFile(String name, boolean mustExist) { } URL resource = resources.iterator().next(); Yaml yaml = new Yaml(); - Map ret = (Map) yaml.load(new InputStreamReader(resource.openStream())); + Map ret = null; + InputStream input = resource.openStream(); + try { + ret = (Map) yaml.load(new InputStreamReader(input)); + } finally { + input.close(); + } if(ret==null) ret = new HashMap(); From 8a9c635f9ffe478f5937ba125201adbf8cb72162 Mon Sep 17 00:00:00 2001 From: yerenkow Date: Fri, 11 Apr 2014 08:49:11 +0300 Subject: [PATCH 333/349] Remove too verbose comment about JAR_JVM_OPTS being array --- bin/storm | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/storm b/bin/storm index 6100f88abf..002467bcc4 100755 --- a/bin/storm +++ b/bin/storm @@ -46,8 +46,6 @@ if (not os.path.isfile(USER_CONF_DIR + "/storm.yaml")): USER_CONF_DIR = CLUSTER_CONF_DIR CONFIG_OPTS = [] CONFFILE = "" - -#note that JAR_JVM_OPTS is now array instead of string JAR_JVM_OPTS = shlex.split(os.getenv('STORM_JAR_JVM_OPTS', '')) From 6c1af3e5b0f78c8d947a731b4609afaff955950c Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Fri, 11 Apr 2014 14:33:54 +0000 Subject: [PATCH 334/349] Updated Changelog STORM-196: When JVM_OPTS are set, storm jar fails to detect storm.jar from environment --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e925c5832..cccda61591 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.9.2-incubating (unreleased) + * STORM-196: When JVM_OPTS are set, storm jar fails to detect storm.jar from environment * STORM-260: Fix a potential race condition with simulated time in Storm's unit tests * STORM-258: Update commons-io version to 2.4 * STORM-270: don't package .clj files in release jars. From 1be3d0f65e4fcb604b14c11a3261aa0cf109ef71 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Fri, 11 Apr 2014 11:20:44 -0400 Subject: [PATCH 335/349] STORM-281: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cccda61591..764b9f4f20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.9.2-incubating (unreleased) + * STORM-281: fix and issue with config parsing that could lead to leaking file descriptors * STORM-196: When JVM_OPTS are set, storm jar fails to detect storm.jar from environment * STORM-260: Fix a potential race condition with simulated time in Storm's unit tests * STORM-258: Update commons-io version to 2.4 From 1a0b46e95ab4ac467525314a75819a75dec92c40 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Fri, 11 Apr 2014 12:11:46 -0400 Subject: [PATCH 336/349] STORM-12: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 764b9f4f20..1fd0e8018b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.9.2-incubating (unreleased) + * STORM-12: reduce thread usage of netty transport * STORM-281: fix and issue with config parsing that could lead to leaking file descriptors * STORM-196: When JVM_OPTS are set, storm jar fails to detect storm.jar from environment * STORM-260: Fix a potential race condition with simulated time in Storm's unit tests From c4ff804c30b331be0af032f01d1b2819d62c7dc2 Mon Sep 17 00:00:00 2001 From: Danijel Schiavuzzi Date: Fri, 11 Apr 2014 18:11:55 +0200 Subject: [PATCH 337/349] Treat command line "-c" option number config values as such (STORM-173) --- storm-core/src/jvm/backtype/storm/StormSubmitter.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/storm-core/src/jvm/backtype/storm/StormSubmitter.java b/storm-core/src/jvm/backtype/storm/StormSubmitter.java index 53c8643155..c7305de02b 100644 --- a/storm-core/src/jvm/backtype/storm/StormSubmitter.java +++ b/storm-core/src/jvm/backtype/storm/StormSubmitter.java @@ -21,11 +21,14 @@ import backtype.storm.utils.BufferFileInputStream; import backtype.storm.utils.NimbusClient; import backtype.storm.utils.Utils; + import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.commons.lang.math.NumberUtils; import org.apache.thrift.TException; import org.json.simple.JSONValue; @@ -78,6 +81,12 @@ public static void submitTopology(String name, Map stormConf, StormTopology topo } stormConf = new HashMap(stormConf); stormConf.putAll(Utils.readCommandLineOpts()); + for (Object confName : stormConf.keySet()) { + String confValue = String.valueOf(stormConf.get(confName)); + if (NumberUtils.isNumber(confValue)) { + stormConf.put(confName, NumberUtils.createNumber(confValue)); + } + } Map conf = Utils.readStormConfig(); conf.putAll(stormConf); try { From 061acf1740d66edd2b90a62635d558a634450c64 Mon Sep 17 00:00:00 2001 From: Derek Dagit Date: Tue, 15 Apr 2014 13:34:13 +0000 Subject: [PATCH 338/349] escape single quotes when logging --- storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj index 1630766eb6..0ffb888204 100644 --- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj +++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj @@ -475,7 +475,9 @@ port worker-id]) command (->> command (map str) (filter (complement empty?))) - shell-cmd (->> command (map #(str \' % \')) (clojure.string/join " "))] + shell-cmd (->> command + (map #(str \' (clojure.string/escape % {\' "\\'"}) \')) + (clojure.string/join " "))] (log-message "Launching worker with command: " shell-cmd) (launch-process command :environment {"LD_LIBRARY_PATH" (conf JAVA-LIBRARY-PATH)}) )) From 3a037463569e81a43b399e4de9ae43f0b5749eff Mon Sep 17 00:00:00 2001 From: Bryan Baugher Date: Tue, 15 Apr 2014 13:10:07 -0500 Subject: [PATCH 339/349] Fixed storm-core shade plugin config --- storm-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storm-core/pom.xml b/storm-core/pom.xml index 8eee3a9945..3329b887b6 100644 --- a/storm-core/pom.xml +++ b/storm-core/pom.xml @@ -244,7 +244,7 @@ org.apache.thrift:* - odg.apache.storm:* + org.apache.storm:* From 442d8e75103f425d211a022184621072326925f0 Mon Sep 17 00:00:00 2001 From: Danijel Schiavuzzi Date: Wed, 16 Apr 2014 18:01:34 +0200 Subject: [PATCH 340/349] Move command line options values introspection logic to Utils#readCommandLineOpts (STORM-173) --- storm-core/src/jvm/backtype/storm/StormSubmitter.java | 6 ------ storm-core/src/jvm/backtype/storm/utils/Utils.java | 10 +++++++++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/storm-core/src/jvm/backtype/storm/StormSubmitter.java b/storm-core/src/jvm/backtype/storm/StormSubmitter.java index c7305de02b..5ef372f1ef 100644 --- a/storm-core/src/jvm/backtype/storm/StormSubmitter.java +++ b/storm-core/src/jvm/backtype/storm/StormSubmitter.java @@ -81,12 +81,6 @@ public static void submitTopology(String name, Map stormConf, StormTopology topo } stormConf = new HashMap(stormConf); stormConf.putAll(Utils.readCommandLineOpts()); - for (Object confName : stormConf.keySet()) { - String confValue = String.valueOf(stormConf.get(confName)); - if (NumberUtils.isNumber(confValue)) { - stormConf.put(confName, NumberUtils.createNumber(confValue)); - } - } Map conf = Utils.readStormConfig(); conf.putAll(stormConf); try { diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java index 7979e4ba15..bc0ba7eb08 100644 --- a/storm-core/src/jvm/backtype/storm/utils/Utils.java +++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java @@ -23,9 +23,11 @@ import backtype.storm.generated.StormTopology; import clojure.lang.IFn; import clojure.lang.RT; + import com.netflix.curator.framework.CuratorFramework; import com.netflix.curator.framework.CuratorFrameworkFactory; import com.netflix.curator.retry.ExponentialBackoffRetry; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.FileOutputStream; @@ -47,7 +49,9 @@ import java.util.Map; import java.util.TreeMap; import java.util.UUID; + import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.math.NumberUtils; import org.apache.thrift.TException; import org.json.simple.JSONValue; import org.yaml.snakeyaml.Yaml; @@ -170,7 +174,11 @@ public static Map readCommandLineOpts() { for (String config : configs) { String[] options = config.split("="); if (options.length == 2) { - ret.put(options[0], options[1]); + Object confValue = options[1]; + if (NumberUtils.isNumber(options[1])) { + confValue = NumberUtils.createNumber(String.valueOf(confValue)); + } + ret.put(options[0], confValue); } } } From a0e2aa6e37ddfd1174c3430499de2b18b34052c5 Mon Sep 17 00:00:00 2001 From: Danijel Schiavuzzi Date: Thu, 17 Apr 2014 10:25:54 +0200 Subject: [PATCH 341/349] Parse command line options as JSON values (STORM-173) --- storm-core/src/jvm/backtype/storm/utils/Utils.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java index bc0ba7eb08..2aa7b3e233 100644 --- a/storm-core/src/jvm/backtype/storm/utils/Utils.java +++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java @@ -172,13 +172,13 @@ public static Map readCommandLineOpts() { commandOptions = commandOptions.replaceAll("%%%%", " "); String[] configs = commandOptions.split(","); for (String config : configs) { - String[] options = config.split("="); + String[] options = config.split("=", 2); if (options.length == 2) { - Object confValue = options[1]; - if (NumberUtils.isNumber(options[1])) { - confValue = NumberUtils.createNumber(String.valueOf(confValue)); + Object val = JSONValue.parse(options[1]); + if (val == null) { + val = options[1]; } - ret.put(options[0], confValue); + ret.put(options[0], val); } } } From b81ba5ee75627312719c713b6c3030b237b342d2 Mon Sep 17 00:00:00 2001 From: Danijel Schiavuzzi Date: Thu, 17 Apr 2014 15:42:20 +0200 Subject: [PATCH 342/349] Clean up imports (STORM-173) --- .../jvm/backtype/storm/StormSubmitter.java | 21 ++++++++++------ .../src/jvm/backtype/storm/utils/Utils.java | 25 +++++++++---------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/storm-core/src/jvm/backtype/storm/StormSubmitter.java b/storm-core/src/jvm/backtype/storm/StormSubmitter.java index 5ef372f1ef..2fd0ce88b3 100644 --- a/storm-core/src/jvm/backtype/storm/StormSubmitter.java +++ b/storm-core/src/jvm/backtype/storm/StormSubmitter.java @@ -17,20 +17,25 @@ */ package backtype.storm; -import backtype.storm.generated.*; -import backtype.storm.utils.BufferFileInputStream; -import backtype.storm.utils.NimbusClient; -import backtype.storm.utils.Utils; - import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.commons.lang.math.NumberUtils; import org.apache.thrift.TException; import org.json.simple.JSONValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import backtype.storm.generated.AlreadyAliveException; +import backtype.storm.generated.ClusterSummary; +import backtype.storm.generated.InvalidTopologyException; +import backtype.storm.generated.Nimbus; +import backtype.storm.generated.StormTopology; +import backtype.storm.generated.SubmitOptions; +import backtype.storm.generated.TopologySummary; +import backtype.storm.utils.BufferFileInputStream; +import backtype.storm.utils.NimbusClient; +import backtype.storm.utils.Utils; /** * Use this class to submit topologies to run on the Storm cluster. You should run your program diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java index 2aa7b3e233..c15e81717d 100644 --- a/storm-core/src/jvm/backtype/storm/utils/Utils.java +++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java @@ -17,23 +17,12 @@ */ package backtype.storm.utils; -import backtype.storm.Config; -import backtype.storm.generated.ComponentCommon; -import backtype.storm.generated.ComponentObject; -import backtype.storm.generated.StormTopology; -import clojure.lang.IFn; -import clojure.lang.RT; - -import com.netflix.curator.framework.CuratorFramework; -import com.netflix.curator.framework.CuratorFrameworkFactory; -import com.netflix.curator.retry.ExponentialBackoffRetry; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.FileOutputStream; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.net.URL; @@ -51,11 +40,21 @@ import java.util.UUID; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.math.NumberUtils; import org.apache.thrift.TException; import org.json.simple.JSONValue; import org.yaml.snakeyaml.Yaml; +import backtype.storm.Config; +import backtype.storm.generated.ComponentCommon; +import backtype.storm.generated.ComponentObject; +import backtype.storm.generated.StormTopology; +import clojure.lang.IFn; +import clojure.lang.RT; + +import com.netflix.curator.framework.CuratorFramework; +import com.netflix.curator.framework.CuratorFrameworkFactory; +import com.netflix.curator.retry.ExponentialBackoffRetry; + public class Utils { public static final String DEFAULT_STREAM_ID = "default"; From 6cc9bfe1c16f668cf8d89ff249136420965e6a51 Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Fri, 18 Apr 2014 18:01:04 +0000 Subject: [PATCH 343/349] Added STORM-285 to Changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fd0e8018b..81a19ab7a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.9.2-incubating (unreleased) + * STORM-285: Fix storm-core shade plugin config * STORM-12: reduce thread usage of netty transport * STORM-281: fix and issue with config parsing that could lead to leaking file descriptors * STORM-196: When JVM_OPTS are set, storm jar fails to detect storm.jar from environment From ae728f81a3f480cc0f02209f5ed1309204a590f0 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Mon, 21 Apr 2014 13:30:41 -0400 Subject: [PATCH 344/349] add storm-kafka files to the binary distribution --- .../binary/src/main/assembly/binary.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/storm-dist/binary/src/main/assembly/binary.xml b/storm-dist/binary/src/main/assembly/binary.xml index 66b6f498a8..6e3670411a 100644 --- a/storm-dist/binary/src/main/assembly/binary.xml +++ b/storm-dist/binary/src/main/assembly/binary.xml @@ -60,15 +60,34 @@ + + + ${project.basedir}/../../external/storm-kafka/target + external/storm-kafka + + storm*jar + + + + ${project.basedir}/../../external/storm-kafka + external/storm-kafka + + README.* + + + + ${project.basedir}/../../examples/storm-starter/target/storm-starter-${project.version}-jar-with-dependencies.jar /examples/storm-starter/ storm-starter-topologies-${project.version}.jar + + From c4ea0caeef0a0923920c3065de997f04240318b7 Mon Sep 17 00:00:00 2001 From: "P. Taylor Goetz" Date: Mon, 21 Apr 2014 15:38:38 -0400 Subject: [PATCH 345/349] STORM-208: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81a19ab7a4..8a758530ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.9.2-incubating (unreleased) + * STORM-208: Add storm-kafka as an external module * STORM-285: Fix storm-core shade plugin config * STORM-12: reduce thread usage of netty transport * STORM-281: fix and issue with config parsing that could lead to leaking file descriptors From 5eeb74f78b03d1d1076bb3c5871481056a5edf7f Mon Sep 17 00:00:00 2001 From: Srinivas Gumdelli Date: Fri, 18 Apr 2014 09:57:41 -0500 Subject: [PATCH 346/349] STORM-288 : Fixes version spelling --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 8894aaf1c4..7e45f3454b 100644 --- a/pom.xml +++ b/pom.xml @@ -153,7 +153,7 @@ 1.4.0 1.1.3 0.3.6 - 2.4 + 2.4 2.5 1.1 0.4.1 @@ -269,7 +269,7 @@ commons-io commons-io - ${commons-io.verson} + ${commons-io.version} org.apache.commons From 8f394953426af6eb827efb2c94b8d786c599a2ca Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Tue, 22 Apr 2014 14:31:30 +0000 Subject: [PATCH 347/349] Added STORM-288: Fixes version spelling in pom.xml --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a758530ec..7ffc08550d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.9.2-incubating (unreleased) + * STORM-288: Fixes version spelling in pom.xml * STORM-208: Add storm-kafka as an external module * STORM-285: Fix storm-core shade plugin config * STORM-12: reduce thread usage of netty transport From 93612f5659b3f2ef176f9f46f470a081cdbaf239 Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Tue, 22 Apr 2014 16:23:47 +0000 Subject: [PATCH 348/349] Updated Changelog for STORM-194 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ffc08550d..f3bcd3a5b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.9.2-incubating (unreleased) + * STORM-194: Support list of strings in *.worker.childopts, handle spaces * STORM-288: Fixes version spelling in pom.xml * STORM-208: Add storm-kafka as an external module * STORM-285: Fix storm-core shade plugin config From 6a08855b48be8208cef5862246342b230d0f68cd Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Tue, 22 Apr 2014 16:41:51 +0000 Subject: [PATCH 349/349] Updated Changelog for STORM-173 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3bcd3a5b3..79b08eb0c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## 0.9.2-incubating (unreleased) + * STORM-173: Treat command line "-c" option number config values as such * STORM-194: Support list of strings in *.worker.childopts, handle spaces * STORM-288: Fixes version spelling in pom.xml * STORM-208: Add storm-kafka as an external module