From 09b2b59968e59124f2feffd91538ebed4d7b2c58 Mon Sep 17 00:00:00 2001 From: Is Isilon Date: Sat, 11 Apr 2015 11:59:41 +0800 Subject: [PATCH] Scaling --- .gitignore | 2 + bin/control | 12 +++ bin/setup | 9 ++ hooks/publish-celery-connection-info | 5 ++ hooks/publish-db-connection-info | 4 + hooks/set-celery-connection-info | 87 ++++++++++++++++++++ metadata/managed_files.yml | 2 - metadata/manifest.yml | 22 ++++- template/.openshift/action_hooks/build | 1 - template/.openshift/action_hooks/deploy | 6 -- template/.openshift/action_hooks/post-deploy | 1 - template/.openshift/action_hooks/post_deploy | 4 - template/.openshift/action_hooks/pre-build | 1 - template/celerytks/__init__.py | 0 template/celerytks/tasks.py | 15 ---- 15 files changed, 139 insertions(+), 32 deletions(-) create mode 100644 .gitignore create mode 100644 bin/setup create mode 100755 hooks/publish-celery-connection-info create mode 100755 hooks/publish-db-connection-info create mode 100755 hooks/set-celery-connection-info delete mode 100644 template/.openshift/action_hooks/build delete mode 100755 template/.openshift/action_hooks/deploy delete mode 100644 template/.openshift/action_hooks/post-deploy delete mode 100755 template/.openshift/action_hooks/post_deploy delete mode 100644 template/.openshift/action_hooks/pre-build delete mode 100644 template/celerytks/__init__.py delete mode 100644 template/celerytks/tasks.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea5cf20 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +env +logs diff --git a/bin/control b/bin/control index 3f7b5c9..ee7d413 100755 --- a/bin/control +++ b/bin/control @@ -1,5 +1,7 @@ #!/bin/bash -e +source $OPENSHIFT_CARTRIDGE_SDK_BASH + PATH=$OPENSHIFT_DATA_DIR:$OPENSHIFT_DATA_DIR/config/:${OPENSHIFT_CELERY_DIR}bin/:${OPENSHIFT_CELERY_DIR}usr/bin:${OPENSHIFT_CELERY_DIR}conf.d:$PATH #TODO add scaling, better status, variable worker names, stop workers properly @@ -39,10 +41,20 @@ function status() { fi } + function catchall { echo "not yet implemented" } +# Ensure arguments. +if ! [ $# -gt 0 ]; then + echo "Usage: $0 [start|restart|stop|status]" + exit 1 +fi + +# Source utility functions. +source $OPENSHIFT_CARTRIDGE_SDK_BASH + case "$1" in start) start ;; stop) stop ;; diff --git a/bin/setup b/bin/setup new file mode 100644 index 0000000..66df359 --- /dev/null +++ b/bin/setup @@ -0,0 +1,9 @@ +#!/bin/bash -e + +source $OPENSHIFT_CARTRIDGE_SDK_BASH + +for dir in logs pid tmp env; do + mkdir -p $dir +done + +mkdir -p $OPENSHIFT_DATA_DIR/.celery diff --git a/hooks/publish-celery-connection-info b/hooks/publish-celery-connection-info new file mode 100755 index 0000000..28ce2bb --- /dev/null +++ b/hooks/publish-celery-connection-info @@ -0,0 +1,5 @@ +#!/bin/bash + +echo "S_CELERY_MASTER=$CELERY_MASTER" +echo "S_CELERY_HOST=$OPENSHIFT_GEAR_DNS" +echo "S_CELERY_PORT=$OPENSHIFT_GEAR_PORT" diff --git a/hooks/publish-db-connection-info b/hooks/publish-db-connection-info new file mode 100755 index 0000000..dc458e8 --- /dev/null +++ b/hooks/publish-db-connection-info @@ -0,0 +1,4 @@ +#!/bin/bash + +echo "OPENSHIFT_CELERY_HOST=$OPENSHIFT_GEAR_DNS" +echo "OPENSHIFT_CELERY_PORT=$OPENSHIFT_GEAR_PORT" diff --git a/hooks/set-celery-connection-info b/hooks/set-celery-connection-info new file mode 100755 index 0000000..4498507 --- /dev/null +++ b/hooks/set-celery-connection-info @@ -0,0 +1,87 @@ +#!/usr/bin/ruby + +def gear_info(tokens, gear_id, &block) + gears = {} + if tokens.length == 1 + gears[gear_id] = tokens.first + else + while not tokens.empty? + gear, delim, data = tokens.shift(3) + data = yield data if block_given? + raise "Invalid data" unless delim == '=' + gears[gear] = data + tokens.shift if tokens.first == ' ' + end + end + gears +end + +def tokenize(s) + tokens = [] + a = "" + state = :start + s.scan(/([ \t]+)|(\\')|(')|([^ \t']+)/) do |args| + space, escaped_delim, delim, text = args + case state + when :start + case + when space then " " + when escaped_delim then raise "Unexpected delimiter" + when delim then state = :within_delim + when text then tokens << text + else raise "error" + end + when :within_delim + case + when space then a << space + when escaped_delim then a << '"' + when delim then tokens << a; a = ""; state = :start + when text then a << text + end + end + end + tokens << a if a.length > 0 + tokens +end + +gear_id = ARGV.shift +domain = ARGV.shift +tokens = tokenize(ARGV.shift) + +# better if this is in creation order (oldest first) +gears = gear_info(tokens, gear_id) do |d| + d.split(' ').map{ |s| s.scan(/\A(.+?)=(.*?);?\Z/).first }.inject({}){ |h, (k,v)| h[k] = v if v != ''; h } +end +gears.each_pair{ |k,v| puts "Found gear #{k}#{k == gear_id ? '* ' : ''} with data #{v.inspect}" } +gear_ids = gears.keys.sort.uniq + +puts "-------" + +was_master = ENV['CELERY_MASTER'] == '1' +masters = gears.map{ |k,v| v['S_CELERY_MASTER'] == '1' ? k : nil }.compact.uniq.sort + +mode = + case ENV['CELERY_MODE'] + when 'read_replica' + if masters.length > 1 + masters = masters[0,1] + elsif masters.length == 0 + masters = gears.keys.uniq.sort[0,1] + end + puts "Running in read replica mode with master #{masters}" + :read_replica + else + masters = gears.keys.uniq.sort + puts "Running sharded with masters #{masters.inspect}" + :sharded + end + +# An array of all of the host:port pairs for the cluster +hosts = gears.map{ |k,v| "#{v['S_CELERY_HOST']}:#{v['S_CELERY_PORT']}" }.compact.uniq.sort +# A list of key value pairs in => for the cluster +members = gears.map{ |k,v| "#{k}=#{v['S_CELERY_HOST']}:#{v['S_CELERY_PORT']}" }.compact.uniq.sort + +File.open('env/CELERY_CLUSTER_MEMBERS', 'w'){ |f| f.puts members.join("\n") } +File.open('env/CELERY_CLUSTER', 'w'){ |f| f.puts hosts.join(",") } +File.open('env/CELERY_CLUSTER_MASTERS', 'w'){ |f| f.puts masters.join(",") } +File.open('env/CELERY_MASTER', 'w'){ |f| f.puts masters.include?(gear_id) ? "1" : "0" } diff --git a/metadata/managed_files.yml b/metadata/managed_files.yml index 24d0b94..be02f60 100644 --- a/metadata/managed_files.yml +++ b/metadata/managed_files.yml @@ -3,5 +3,3 @@ locked_files: - env/ - env/* - conf.d/* -processed_templates: -- '**/*.erb' diff --git a/metadata/manifest.yml b/metadata/manifest.yml index 45da873..633bec2 100644 --- a/metadata/manifest.yml +++ b/metadata/manifest.yml @@ -2,7 +2,7 @@ Name: celeryd Cartridge-Short-Name: CELERY Display-Name: Celery Cartridge Description: A cartridge for running the celeryd daemon to allow asynchronous tasks in your application. -Version: '0.15' +Version: '0.16' License: ASL 2.0 Vendor: Matti HEIKKILA Cartridge-Version: @@ -12,9 +12,18 @@ Website: https://github.com/wassname/openshift-celery-cartridge Source-Url: https://github.com/wassname/openshift-celery-cartridge.git Categories: - service -Requires: python-2.7 + - embedded +Requires: celery +Scaling: + Min: 1 + Max: -1 Provides: - celery +Endpoints: + - Private-IP-Name: HOST + Private-Port-Name: PORT + Private-Port: 16388 + Public-Port-Name: PROXY_PORT Cart-Data: - Key: OPENSHIFT_CELERY_HOME Type: environment @@ -28,3 +37,12 @@ Cart-Data: Group-Overrides: - components: - celery +Subscribes: + set-redis-connection-info: + Type: "ENV:NET_TCP:db:celery:connection-info-v1" + Required: false +Publishes: + publish-db-connection-info: + Type: "ENV:NET_TCP:db:connection-info" + publish-redis-connection-info: + Type: "ENV:NET_TCP:db:celery:connection-info-v1" diff --git a/template/.openshift/action_hooks/build b/template/.openshift/action_hooks/build deleted file mode 100644 index 8b13789..0000000 --- a/template/.openshift/action_hooks/build +++ /dev/null @@ -1 +0,0 @@ - diff --git a/template/.openshift/action_hooks/deploy b/template/.openshift/action_hooks/deploy deleted file mode 100755 index 3875983..0000000 --- a/template/.openshift/action_hooks/deploy +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# This deploy hook gets executed after dependencies are resolved and the -# build hook has been run but before the application has been started back -# up again. This script gets executed directly, so it could be python, php, -# ruby, etc. - diff --git a/template/.openshift/action_hooks/post-deploy b/template/.openshift/action_hooks/post-deploy deleted file mode 100644 index 8b13789..0000000 --- a/template/.openshift/action_hooks/post-deploy +++ /dev/null @@ -1 +0,0 @@ - diff --git a/template/.openshift/action_hooks/post_deploy b/template/.openshift/action_hooks/post_deploy deleted file mode 100755 index d38c5da..0000000 --- a/template/.openshift/action_hooks/post_deploy +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -source $OPENSHIFT_HOMEDIR/python/virtenv/bin/activate -export PYTHONPATH=`echo $OPENSHIFT_REPO_DIR`:$PYTHONPATH diff --git a/template/.openshift/action_hooks/pre-build b/template/.openshift/action_hooks/pre-build deleted file mode 100644 index 8b13789..0000000 --- a/template/.openshift/action_hooks/pre-build +++ /dev/null @@ -1 +0,0 @@ - diff --git a/template/celerytks/__init__.py b/template/celerytks/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/template/celerytks/tasks.py b/template/celerytks/tasks.py deleted file mode 100644 index ac79360..0000000 --- a/template/celerytks/tasks.py +++ /dev/null @@ -1,15 +0,0 @@ -import celery - -app = celery.Celery() -app.config_from_object('celeryconfig') - -@app.task -def addtest(x, y): - return x + y - -@app.task -def run_spiders(name, **kwargs): - """ - @param name: spider name - """ - pass