google “chef-solo rails deploy git submodule checkout branch sha”

Working with chef resource deploy, faced several issues with Git provider, for examples

– enable_submodule does not work properly
– it does not get the latest code from submodules
– it does not allow you to choose specific branch and sha of any submodule
– in fact for main repo it does not allow you to switch to any specific branch and sha
– for some reason it says branch=version, who knows why ….

I tried correcting all above, and below is one of my examples, its working for me


# solo.rb
require File.expand_path('../lib/global_constants.rb', __FILE__)
require File.expand_path('../lib/git_addition.rb', __FILE__)
root = File.absolute_path(File.dirname(__FILE__))
file_cache_path root
cookbook_path [ root + '/cookbooks', root + '/custom_cookbooks']
data_bag_path root + '/data_bags'
node_path root + '/nodes'
role_path root + '/roles'
#log_level :debug
#log_location STDOUT
#verbose_logging false
# lib/git_addition.rb
require 'chef/log'
require 'chef/provider'
require 'chef/mixin/shell_out'
require 'fileutils'
require 'shellwords'
class Chef
class Provider
class Git < Chef::Provider
def fetch_updates
#setup_remote_tracking_branches if @new_resource.remote != 'origin'
converge_by("fetch updates for #{@new_resource.remote}") do
fetch_command = "rm Gemfile.lock; git fetch #{@new_resource.remote} && git fetch #{@new_resource.remote} –tags && git checkout #{@new_resource.branch} && git reset –hard #{@new_resource.sha}"
Chef::Log.debug "Fetching updates from #{new_resource.remote} and resetting to revision #{target_revision}"
shell_out!(fetch_command, run_options(:cwd => @new_resource.destination))
end
end
def checkout
sha_ref = target_revision
converge_by("checkout ref #{sha_ref} branch #{@new_resource.branch}") do
shell_out!("rm Gemfile.lock; git fetch -u && git checkout #{@new_resource.branch} && git reset –hard #{@new_resource.sha} ", run_options(:cwd => @new_resource.destination))
Chef::Log.info "#{@new_resource} checked out branch: #{@new_resource.branch} reference: #{sha_ref}"
end
end
def target_revision
@target_revision ||= begin
if sha_hash?(@new_resource.sha)
@target_revision = @new_resource.sha
else
@target_revision = remote_resolve_reference
end
end
end
alias :revision_slug :target_revision
end
end
end
class Chef
class Resource
class Deploy < Chef::Resource
attr_accessor :sha
def sha(arg=nil)
set_or_return(
:sha,
arg,
:kind_of => [ String ]
)
end
end
end
end
# main recipe
app_root = "#{$pw_home_dir}/myapp-deploy"
current_path = "#{app_root}/current"
cache_path = "#{app_root}/shared/cached-copy"
package "g++"
package "nodejs"
package "mysql-client"
package "libmysqlclient-dev"
rvm_ruby node.rvm_ruby
rvm_default_ruby node.rvm_ruby
deploy_revision app_root do
action $chef_rails_deploy
user node.user
group node.group
repo node.repo
branch ($cfg["myapp_branch"] or node.myapp_branch)
sha ($cfg["myapp_sha"] or node.myapp_sha)
env = { "RAILS_ENV" => $rails_env }
Chef::Mixin::DeepMerge.merge(node.environment, env)
environment env
before_migrate do
myapp_config cache_path
myapp_checkout cache_path
rvm_shell "bundle" do
ruby_string node.rvm_ruby
cwd cache_path
user node.user
group node.group
code %{ bundle –path #{cache_path} –gemfile #{cache_path}/Gemfile }
end
end
purge_before_symlink %w{tmp}
create_dirs_before_symlink %w{uploads tmp}
symlinks {}
symlink_before_migrate({
"log" => "log",
"tmp" => "tmp",
"public" => "public",
"cached-copy/config/database.yml" => "config/database.yml",
"cached-copy/config/config.yml" => "config/config.yml"
})
migrate true
migration_command "echo 'skipping RAILS_ENV=#{$rails_env} bundle exec rake db:migrate –trace' "
shallow_clone true # TBD
keep_releases 10
before_restart do
myapp_checkout "#{app_root}/current"
#assets ($cfg["myapp-revision"] or "HEAD") do
# app_path cache_path
#end
if production?
#rvm_gem "passenger" do
# options " -y "
#end
# follow myapp/installprofile.apache
#restart_command "touch tmp/restart.txt"
end
end
end
# definitions/submodule.rb
define :submodule, :name=>nil do
return if rollback?
rvm_shell "submodule #{params[:name]}" do
ruby_string node.rvm_ruby
cwd params[:name]
user node.user
group node.group
code %{ git submodule init && git submodule update }
end
end
# definitions/checkout.rb
define :checkout, :name=>nil, :cwd=>nil, :branch=>nil, :sha=>nil do
rvm_shell "#{params[:name]} #{params[:branch]} #{params[:sha]} => #{params[:cwd]}" do
ruby_string node.rvm_ruby
cwd params[:cwd]
user node.user
group node.group
code %{ git checkout #{params[:branch]} && git pull && git reset –hard #{params[:sha]} }
end
end
# definitions/myapp_checkout.rb
define :myapp_checkout, :name=>nil do
return if rollback?
path = params[:name]
submodule path
checkout "sm1" do
cwd "#{path}/vendor/plugins/sm1"
branch ($cfg["myapp_sm1_branch"] or node.myapp_sm1_branch)
sha ($cfg["myapp_sm1_sha"] or node.myapp_sm1_sha)
end
checkout "sm2" do
cwd "#{path}/vendor/plugins/sm2"
branch ($cfg["myapp_sm2_branch"] or node.myapp_sm2_branch)
sha ($cfg["myapp_sm2_sha"] or node.myapp_sm2_sha)
end
end

view raw

default.rb

hosted with ❤ by GitHub

DSL approach made it really simple

How much watts we do consume per cycle? Have we joined any of the green innovations like for instance green button. It would be great to know how much watts my house consumed this cycle, will I be able to reduce it, any one out there to help me reduce it. I have so many appliances running at home and I do have a smart meter measuring how much has been consumed. Would you also want to know how does meter measure your watts consumed, and how smart meters provide data for further calculation of your electricity bills.




There are several initiatives taking place in US, PlotWatt, FirstFuel and more, and they help you know your own detailed energy usage, along with other features like calculations, helping you out choosing right pricing plan, giving you several suggestions on how could you reduce your consumption and electricity bills.

Okeh, so this was a quick about Green Button.

Working on one of such initiatives, I came across to define a solution that will enable users to see what electricity bill they would end up paying if they apply some different pricing plan, hence ability to apply different pricing plans against the data consumed for the current or past cycles and being able to compare plans. Rather this would actually help utilities defining their trial plans.

Being rubiest for now more than 5 years, it immediately struck me about having a nice ruby DSL for this one, which could be easier to extend for new rules or sub-requirements. And this was a too much of excitement for me. Writing DSL gives you opportunities to play around ruby mixins, better modeling your classes, modules, beautifully engaging them with each other to work for us and present a nice output.

form

But there was a bad news for me, team decided to do this in python :(  … though still I am to design and develop it.

Not having much exposure to python, having my ruby design plan ready on my mind, I started looking similar python capabilities, parsing, meta-programming, DSL models and more, team as well decided an approach that no need admin to define plans on the fly( like one above), but we could have some fixed plans defined, that actually saved my efforts.

Following is one example of one of those plans we have,


from calc import Calc
from plan import Plan
from datetime import datetime
….
class OurTrialPlan(Plan):
def __init__(self, start_date, end_date, usage, extra=None):
super(OurTrialPlan, self).__init__()
self.calc = Calc(start_date, usage)
self._extra = extra if type(extra)==dict else {}
def calculate(self):
….
add(10, 'customer charges')
# OFF-PEAK
usage = for_month_of(['Jan', 'Feb', 'Oct'])
cost, tier_rate = slab_wise_tariff( usage, no_change_rates)
add( cost, 'no_change', rate=tier_rate)
# KU OFF-PEAK
usage = for_month_of(['Mar', 'Apr', 'May', 'Nov', 'Dec']) and during('10pm','6am')
add( usage*0.0265, 'ke_flat_rate', rate=0.0265)
# KU ON-PEAK
usage = for_month_of(['Mar', 'Apr', 'May', 'Nov', 'Dec']) and during('6am','10pm')
cost, tier_rate = slab_wise_tariff( usage, ke_base_rates)
add( cost, 'ke_base_rates', rate=tier_rate)
cost, tier_rate = slab_tariff( usage, ke_surcharge)
add( cost, 'ke_surcharge', rate=tier_rate)
…..
return tariff()

view raw

TrialPlan.py

hosted with ❤ by GitHub

And we do have several such plans and we could define more, simply focusing on plan rules. The guts have been encapsulated using classes Plan and Calc. Calc helps to collect, compute usage for the specified period, while Plan provides semantics  around.

I did enjoy python scripting a lot, python looks to me as a raw scripting platform where ruby as a very nice, user-friendly scripting platform.

dealing with concerns

Taking care of concerns, my views are

They are meant not to have large/GOD classes

They are not specific to ActiveRecord

They are there to help us out find out dependencies easier

They should not be dependent on each other, if module A has concern :b, then there should exist paths “root/a.rb” and “/root/a/b.rb“. Though this we ourself have to make sure about concerns path by writing additional code at initialization

I always willing to write clean and neat code, and have end up maintaining my concerns the following ways

  1. Reopening Class
  2. Include Module
  3. Extend ActiveSupport::Concern

I do have one doubt though, out of 3 approaches, which one is good for faster execution?


# config/initializers/concerns.rb
class << ActiveRecord::Base
def concerned_with(*concerns)
concerns.each do |concern|
require_dependency "#{name.underscore}/#{concern}"
klass = concern.to_s.classify.constantize rescue nil
send(:include, klass) if klass.is_a? Module
end
end
end
# Approach #1 opening User class
# app/models/user/validations.rb
class User << ActiveRecord::Base
validates :mobile, :length => { :in => 6..20 }, :allow_blank => true, :allow_nil => true
validates_presence_of :country, :name
before_create :is_zip_validated_by_yahoo?
# and so on
end
# Approach #2 Let User include module
# app/models/user/module_name.rb
module ModuleName
def blah_blah_blah
end # and so on ….
end
# Approach #3 extend ActiveSupport::Concern
# app/models/user/ensure_something_type.rb
module EnsureSomethingType
extend ActiveSupport::Concern
included do
before_create :ensure_something_type
belongs_to :something_type
has_one :something_base_type, :through => :something_type
end
def ensure_something_type
self.something_type ||= SomethingType.find_by_name("something something")
end
end
# app/models/user.rb
  concerned_with  :validations,
    :data_uploader,
    :ensure_something_type,
    :jingo_filters,
    :macro_reports,
    :pricing,

view raw

concerns.rb

hosted with ❤ by GitHub

handle_redirect_loop_if_appears


class ApplicationController < ActionController::Base
before_filter :handle_redirect_loop_if_appears
after_filter :set_session_back_url
def handle_redirect_loop_if_appears
return session[:redirect_loop_count] = nil unless request_repeats?
return session[:redirect_loop_count] = 1 unless session[:redirect_loop_count]
max_observed_time_diff_betn_reqs_when_in_loop = 10000
max_loop_count = 3
session[:last_request_time] = Time.now.to_i
session[:redirect_loop_count] += 1
return if session[:redirect_loop_count] < max_loop_count
if last_request_after < max_observed_time_diff_betn_reqs_when_in_loop
session[:last_request_time] = nil
session[:redirect_loop_count] = nil
redirect_to rescue_url and return
end
end
def last_request_after
return unless session[:last_request_time]
time_diff = Time.now.to_isession[:last_request_time].to_i
time_diff = time_diff * (1) if time_diff < 0
end
def rescue_url
home_url
end
def home_url
'/'
end
def request_repeats?
session[:back_url] == request.fullpath
end
def set_session_back_url
return unless request.get?
return if request.xhr?
session[:back_url] = request.fullpath
end
end

mailto:jenkins,subject:test case1, case2 on http://staging.site.com

mailto:jenkins,subject:test case1, case2 on http://staging.site.com

So this is something we thought of

– email Jenkins
– to run one or more jobs
– against one of our deployed staging apps(given base_url)

We wrote a mail_watcher.rb, which periodically reads such emails, and rewrite <JENKINS_CHECKS_DIR>/<test_case_name>.checks files adding parameters to be passed on to the test case. In this case it is BASE_URL=. And all we know that we could make Jenkins

1. watch files getting modified
2. use file contents(key=val), as paramters further to be used or passed on to the job exec

Ohh, wait, I did not know how could Jenkins read parameters from a given key=val file and pass it on further. We actaully wrote another shell script which runs as the first step of every job and reads the testcase_name.checks file and set ENV variables like for example BASE_URL.

So in case of subject:test testcase1 on http://mysite.com

– mail watcher will write testcase1.checks file with BASE_URL=http://mysite.com
– shell script will set ENV variable BASE_URL=http://mysite.com


Following is the mail watcher that we simply run to be able to get Jenkins job triggered.


require 'net/imap'
class MailWatcher
Server = 'imap.gmail.com'
Username = ''
Password = ''
Folder = 'INBOX'
PollingInterval = 60
CheckFileLocation = "#{File.dirname(__FILE__)}/../../tmp/checks/"
def fetch
@imap = Net::IMAP.new Server, 993, true
@imap.login Username, Password
@imap.select Folder
@imap.search(["NOT", "SEEN"]).each do |message_id|
env = @imap.fetch(message_id, "ENVELOPE")[0].attr["ENVELOPE"]
@subject = env.subject
return unless @subject.match(/^test .+/)
test_cases.each do |test_case_name|
File.open("#{CheckFileLocation}#{test_case_name}.check",'w') { |file| file.puts base_url }
end
mark_delete message_id
end
@imap.logout
@imap.disconnect
end
def start_watching
while(1) do
fetch
sleep PollingInterval
end
end
def base_url
@base_url ||= "base_url=#{@subject.match(/ on (https?\:\/\/[a-z\.]+)$/)[1]}" rescue 'base_url='
end
def test_cases
@test_cases = @subject.split(' on http').first[5..-1].strip.split(',').collect{ |test_case| test_case.strip.gsub(' ','_')}
end
def mark_delete message_id
@imap.store(message_id, "+FLAGS", [:Seen])
@imap.store(message_id, "+FLAGS", [:Deleted])
end
end
Process.daemon(true)
MailWatcher.new.start_watching

view raw

mail_watcher.rb

hosted with ❤ by GitHub

Now any of us teammates, can anytime send an email and get a test case executed. In addition we are as well generating a detailed report email for each job-run that gets published on a Google group. We do have our own custom, half-baked, reporting mechanism(ruby code), which we want to compile the job-execution-report email for each testcase/job executed.

Example reports

subject: testcase1 failed.
body: <Jenkins console output>

subject: rspec 231 passed, 2 failed.
body: <Jenkins console output>

reading huge xml in ruby

Dealing with huge xml? say of size 400 MB or more than a GB. No worries, Nokogiri::XML::Reader is there for your help. But let me tell you it takes a while before you understand it and start using it. Needs to understand types of elements, and when to read value and and when to read attributes. However I have went over it and tried defining my own module which now simplifies it.

Your large xml doc must be having thousands of records, and you may or may not know the structure of record. This module will help you easily read your records, its elements, something like this below …

hey_my_module
  take_this_xml
  and look_for_these_elements
  let_me_know_if_you_find_any_of_them

…hence

HugeXML.read xml, elements_lookup do |element|
  # => element{ :name, :value, :attributes}
end

so simple right? wanna check it out… here it is  => github.com/amolpujari/reading-huge-xml

 

DRYing if @current_user.nil?

While working on one of rails applications, as I usually tend to check whether user is logged in, specially in views as follows

<% if @current_user.nil? %>
  <a href='url to login page'>
<% else %>
  <a href='some url where logged in user have access'>
<% end %>

And over some period I found this pattern repeating everywhere.

I thought of DRYing it, I wanted to have a kind of before_filter for every link that appear on the page, every link should go for login if that page or feature require user entity. and this should be handled by a common thing, similar to before_filter: login_require, on server side. I intend to achieve similar in views. A while back when I read up about html5 custom data attribute, and unobtrusive javascript etc, referring to them, I got one idea as described below, though it is not something that I am making use of custom data attribute exactly, but I found this idea useful and working. May be someone would have good suggestions here for me.

I will put any link which require user login as follows

<a href="/feature/page/" login-require >Feature</a>

And then I will have my after_load.js which will have the following event binding

jQuery('a[login-require]').bind('click', function() {
  <% if @current_user.nil? %>
    openOverlay('overlay-login');
    return false;
  <% else %>
    return true;
  <% end %>
});

to_wordpress_html ruby_file_path: Code highlighter for wordpress

Started tech blogging and found it is necessary to have a nice code highlighter, and I was actually looking out for a blogging tool that would appear very simpler, nice, less time to post, and still have good control over minor aspects.  Trying to achieve all this using wordpress, and found a good friend coderay (when looked inside html from m.onkey.org).

Now.. to have any code snippet here on the blog I just have to say “to_wordpress_html” as follows

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
def to_wordpress_html file_path
  return unless file_path

  require 'coderay'
  html = CodeRay.scan_file( file_path, :ruby).div(:line_numbers => :table)

  html = "<div style=\"overflow:auto;margin-top: 5px;\">#{html}</div>"

  html.sub!('class="code"><pre>', ' class="code"><pre style="padding-left: 2px;margin-bottom: 0;padding-bottom: 0;"> ')
  html.sub!('<pre><a href="#n1" ',' <pre style="border-right: 1px solid gray;padding-left: 2px;padding-right: 2px;margin-bottom: 0;padding-bottom: 0;"><a href="#n1" ')

  html.sub!('class="CodeRay"',      ' class="CodeRay" style="margin-bottom:0;" ')
  html.sub!('class="line-numbers"', ' class="line-numbers" style="padding:0;" ')
  html.sub!('class="code"',         ' class="code" style="padding:0;" ')
end  

if not ARGV[0]
  puts "Require ruby file."
else
  File.open('code.html','w'){ |file| file.write to_wordpress_html(ARGV[0]) }
end

# ruby coderay_for_wordpress.rb coderay_for_wordpress.rb

May be I will later add support for lines from and to, and more…

adding to my migration helper

Recently I added few methods to my migration helpers for the reasons

  1. Rails support for many databases causes not to have foreign key support, which I wanted to have
  2. Why to use default SIGNED column for id or for other column where it is not needed, so getting more numbers store
  3. One may not want to have unicode support for system managed tables(with no user generated content), unicode table requires more space.

I tested below units against mysql 5.1 only

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
# migration_helpers.rb
module MigrationHelpers

  def add_foreign_key(from_table, from_column, to_table, to_column='id')
    constraint_name = "fk_#{from_table}_#{from_column}"
    #execute %{alter table #{from_table} add index (#{from_column})}
    add_index from_table, from_column
    execute %{alter table #{from_table} add constraint #{constraint_name} foreign key (#{from_column}) references #{to_table}(#{to_column}) ON DELETE NO ACTION ON UPDATE NO ACTION}
  end

  def remove_foreign_key(from_table, from_column)
    constraint_name = "fk_#{from_table}_#{from_column}"
    execute %{alter table #{from_table} drop foreign key #{constraint_name}}
    #execute %{alter table #{from_table} drop index index_on_#{from_table}_on_#{from_column}} # column needs to be dropped first
  end

  def innodb_latin
    'ENGINE = InnoDB DEFAULT CHARACTER SET = latin1'
  end

  def id_column
    'INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY (id)'
  end

  def foreign_key_column
    'INT UNSIGNED'
  end

  def count_coulmn
    'INT UNSIGNED DEFAULT 0'
  end

  def number_column
    'INT UNSIGNED'
  end
end

 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
class CreateAuthentications < ActiveRecord::Migration
  def self.up
    create_table :authentications, :id => false, :options => innodb_latin do |t|
      t.column      :id, id_column
      t.column      :user_id, foreign_key_column
      t.string      :provider
      t.string      :uid
      t.text        :user_data
      t.timestamps
    end
  end

  def self.down
    drop_table :authentications
  end
end