#!/usr/bin/env oo-ruby

#--
# Copyright 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#++

require 'rubygems'
require 'getoptlong'
require 'time'
require "/var/www/openshift/broker/config/environment"
# Disable analytics for admin scripts
Rails.configuration.analytics[:enabled] = false
Rails.configuration.msg_broker[:rpc_options][:disctimeout] = 20
Rails.configuration.msg_broker[:rpc_options][:timeout] = 600

def usage_additional_plugins
  usage_info = ""
  if $billing_enabled
    billing_api = OpenShift::BillingService.instance
    usage_info += billing_api.display_check_help.to_s
  end
  usage_info
end

def usage
  puts <<USAGE
== Synopsis

#{$0}: Check all user applications

== Usage

#{$0} OPTIONS

Options:
-v|--verbose
    Print information about each check being performed
-l|--level [0, 1]
    Level '0' is default, and checks to see if any gears are present in mongo but don't exist on the nodes and vice-versa.
    Level '1' performs checks for consumed_gears count mismatches, application data integrity in mongo, and checks for unused and unreserved gear UIDs
    #{usage_additional_plugins}
-h|--help
    Show Usage info
USAGE
  exit 255
end

args = {}
begin
  opts = GetoptLong.new(
    ["--verbose",          "-v", GetoptLong::NO_ARGUMENT],
    ["--level",            "-l", GetoptLong::REQUIRED_ARGUMENT],
    ["--help",             "-h", GetoptLong::NO_ARGUMENT]
  )
  opts.each{ |k,v| args[k]=v }
rescue GetoptLong::Error => e
  usage
end

level = args["--level"].to_i || 0
verbose = args["--verbose"]
usage if args["--help"]

def datastore_has_gear?(gear_uuid, app_id=nil, false_positive_response=true)
  $false_positive_check_cnt += 1
  if $false_positive_check_cnt < FALSE_POSITIVE_CHECK_LIMIT
    # specifying the app_id first, if present
    # this makes better use of the indexes and performance is better 
    if app_id
      query = { "_id" => app_id }
    else
      query = {}
    end
    query['group_instances.gears.uuid'] = gear_uuid
    return Application.where(query).exists?
  else
    return false_positive_response
  end
end

def datastore_has_gear_uid?(gear_uid, si_list, false_positive_response=true)
  $false_positive_check_cnt += 1
  if $false_positive_check_cnt < FALSE_POSITIVE_CHECK_LIMIT
    query = {"group_instances.gears" => {"$elemMatch" => {"uid" => gear_uid, "server_identity" => {"$in" => si_list}}}}
    return Application.where(query).exists?
  else
    return false_positive_response
  end
end

def district_has_available_uid?(district_id, gear_uid, false_positive_response=true)
  $false_positive_check_cnt += 1
  if $false_positive_check_cnt < FALSE_POSITIVE_CHECK_LIMIT
    query = { "_id" => district_id.to_s, "available_uids" => gear_uid }
    return District.where(query).exists?
  else
    return false_positive_response
  end
end

def get_user_info(user, level=0)
  user_ssh_keys = []
  user["ssh_keys"].each { |k| user_ssh_keys << "#{user['_id'].to_s}-#{k['name']}#{SSH_KEY_COMMENT_DELIMITER}#{Digest::MD5.hexdigest(k["content"])}" if k["content"] } if user["ssh_keys"]
  user_caps = user["capabilities"]
  if user["plan_id"]
    plan_caps = OpenShift::BillingService.instance.get_plans[user["plan_id"].to_sym][:capabilities]
    user_caps = plan_caps.deep_dup.merge(user_caps)
  end
  user_info = {"consumed_gears" => user["consumed_gears"], "domains" => {}, "login" => user["login"], "ssh_keys" => user_ssh_keys,
               "max_untracked_addtl_storage_per_gear" => user_caps["max_untracked_addtl_storage_per_gear"],
               "plan_id" => user["plan_id"], "plan_state" => user["plan_state"], "usage_account_id" => user["usage_account_id"],
               "parent_user_id" => user["parent_user_id"]}
  user_info
end

def check_consumed_gears(user_id)
  begin
    actual_gears = 0
    user = CloudUser.find_by(:_id => user_id)
    user.domains.each do |d|
      d.applications.each do |a|
        a.group_instances.each do |gi|
          actual_gears += gi.gears.length
        end
      end
    end
    return user.consumed_gears, actual_gears
  rescue Mongoid::Errors::DocumentNotFound
    puts "Error: User with ID #{user_id} not found in mongo"
    return 0, 0
  end
end

def get_premium_carts
  return $premium_carts unless $premium_carts.nil?

  premium_carts = []
  all_carts = CartridgeCache.cartridges
  all_carts.each do |cart|
    premium_carts.push(cart.name) if cart.is_premium?
  end
  $premium_carts = premium_carts

  return $premium_carts
end

def assign_usage_hash(urec, gear_hash, storage_hash, cart_hash, summary, verbose)
  gear_id = urec['gear_id'].to_s
  if !UsageRecord::EVENTS.values.include?(urec['event'])
    puts "FAIL" if verbose
    summary << "Found record in usage_records collection with invalid event '#{urec['event']}' for gear Id '#{gear_id}'."
    return
  end

  hash = nil
  case urec['usage_type']
  when UsageRecord::USAGE_TYPES[:gear_usage]
    hash = gear_hash
  when UsageRecord::USAGE_TYPES[:addtl_fs_gb]
    hash = storage_hash
  when UsageRecord::USAGE_TYPES[:premium_cart]
    hash = cart_hash
  else
    puts "FAIL" if verbose
    summary << "Found invalid usage type '#{urec['usage_type']}' in usage_records collection for gear Id '#{gear_id}'."
    return
  end

  unless hash[gear_id]
    hash[gear_id] = {}
    hash[gear_id]['num_begin_recs'] = 0
    hash[gear_id]['num_end_recs'] = 0
  end
  if hash[gear_id]['created_at'].nil? or (urec['created_at'] > hash[gear_id]['created_at'])
    hash[gear_id]['app_name'] = urec['app_name']
    hash[gear_id]['created_at'] = urec['created_at']
  end
  if urec['event'] == UsageRecord::EVENTS[:end]
    hash[gear_id]['num_end_recs'] += 1
  else
    hash[gear_id]['num_begin_recs'] += 1
  end 
  if urec['cart_name']
    hash[gear_id]['cart_name'] = [] unless hash[gear_id]['cart_name']
    hash[gear_id]['cart_name'] << urec['cart_name']
  end
  if urec['addtl_fs_gb']
    hash[gear_id]['addtl_fs_gb'] = urec['addtl_fs_gb']
  end
end

def process_usage_hash(gear_hash, storage_hash, cart_hash)
  [gear_hash, storage_hash, cart_hash].each do |hash|
    hash.delete_if do |gear_id, gear_info|
      gear_info['num_begin_recs'] == gear_info['num_end_recs']
    end
  end
end

FALSE_POSITIVE_CHECK_LIMIT = 4000
SSH_KEY_COMMENT_DELIMITER = "::"
$false_positive_check_cnt = 0
$premium_carts = nil
$districts_enabled = Rails.configuration.msg_broker[:districts][:enabled]
$billing_enabled = Rails.configuration.respond_to?('billing')
$first_district_uid = Rails.configuration.msg_broker[:districts][:first_uid]

summary = []

datastore_hash = {}
user_hash = {}
domain_hash = {}
domain_namespace_hash = {}
gear_uid_hash = {}
gear_id_hash = {}
district_hash = {}
gear_sshkey_hash = {}

# store the current time for comparisons
current_time = Time.now

puts "Started at: #{current_time}"
start_time = (current_time.to_f * 1000).to_i

if level >= 1
  user_selection = {:fields => ["consumed_gears", "login", "ssh_keys.name", "ssh_keys.content",
                                "capabilities.max_untracked_addtl_storage_per_gear", "plan_id",
                                "plan_state", "usage_account_id", "parent_user_id"], :timeout => false}
  user_selection_primary = user_selection.dup
  user_selection_primary[:read] = :primary

  domain_selection = {:fields => ["owner_id", "system_ssh_keys.name", "system_ssh_keys.content", "canonical_namespace", "members"],
                      :timeout => false}

  OpenShift::DataStore.find(:cloud_users, {}, user_selection) do |user|
    user_hash[user["_id"].to_s] = get_user_info(user, level)
  end

  OpenShift::DataStore.find(:domains, {}, domain_selection) do |domain|
    owner_id = domain["owner_id"].to_s
    domain_hash[domain["_id"].to_s] = owner_id
    domain_namespace_hash[domain["_id"].to_s] = domain['canonical_namespace']
    system_ssh_keys = []
    domain["system_ssh_keys"].each { |k| system_ssh_keys << "domain-#{k['name']}#{SSH_KEY_COMMENT_DELIMITER}#{Digest::MD5.hexdigest(k["content"])}" if k["content"] } if domain["system_ssh_keys"]

    if !user_hash[owner_id]
      OpenShift::DataStore.find(:cloud_users, {"_id" => domain["owner_id"]}, user_selection_primary) do |user|
        user_hash[user["_id"].to_s] = get_user_info(user, level)
      end
    end

    if domain['members'].nil? or domain['members'].empty?
      summary << "Domain '#{domain['_id']}' has no members in mongo."
    end

    if user_hash[owner_id]
      user_hash[owner_id]["domains"][domain["_id"].to_s] = 0
      user_hash[owner_id]["ssh_keys"] |= system_ssh_keys
    else
      summary << "User '#{owner_id}' for domain '#{domain['_id']}' does not exist in mongo."
    end
  end
end

if level >= 1 and $districts_enabled
  OpenShift::DataStore.find(:districts, {}, {:timeout => false}) do |district|
    si_list =  district["server_identities"].map {|si| si["name"]}
    si_list.delete_if {|si| si.nil?}
    district_hash[district["_id"].to_s] = [ district["name"], district["max_capacity"], si_list, district["available_uids"] ]

    # check available_uids list length and available_capacity
    if district["available_uids"].length != district["available_capacity"]
      summary << "District '#{district["name"]}' has (#{district["available_uids"].length}) available UIDs but (#{district["available_capacity"]}) available capacity"
    end
  end
end

app_selection = {:fields => ["name", "created_at", "domain_id", "group_instances.gears._id", "group_instances.gears.app_dns",
                             "group_instances.gears.uuid", "group_instances.gears.uid", "group_instances.gears.server_identity",
                             "group_instances._id", "component_instances._id", "component_instances.cartridge_name",
                             "component_instances.group_instance_id", "app_ssh_keys.name", "app_ssh_keys.content",
                             "group_overrides.additional_filesystem_gb", "group_overrides.components.cart",
                             "members", "domain_namespace", "owner_id"],
                 :timeout => false}
app_query = {"group_instances.gears.0" => {"$exists" => true}}
OpenShift::DataStore.find(:applications, app_query, app_selection) do |app|
  gear_count = 0
  owner_id = nil
  login = nil
  has_gears = false
  has_dns_gear = false
  creation_time = app['created_at']
  domain_id = app['domain_id'].to_s
  app_ssh_keys = []
  app['app_ssh_keys'].each { |k| app_ssh_keys << "#{k['name']}#{SSH_KEY_COMMENT_DELIMITER}#{Digest::MD5.hexdigest(k["content"])}" if k["content"] } if app['app_ssh_keys']

  if level >= 1
    owner_id = domain_hash[domain_id]

    if owner_id.nil?
      if (current_time - creation_time) > 600
        summary << "Application '#{app['name']}' does not have a domain '#{domain_id}' in mongo."
      end
    elsif user_hash[owner_id].nil?
      if (current_time - creation_time) > 600
        summary << "Application '#{app['name']}' for domain '#{domain_id}' does not have a user '#{owner_id}' in mongo."
      end
    else
      login = user_hash[owner_id]["login"]
      app_ssh_keys |= user_hash[owner_id]["ssh_keys"]

      if app['owner_id'].nil?
        summary << "Application '#{app['name']}' for domain '#{domain_id}' does not have the denormalized owner_id set in mongo."
      elsif app['owner_id'].to_s != owner_id
        summary << "Application '#{app['name']}' for domain '#{domain_id}' has a denormalized owner_id of '#{app['owner_id']}' instead of the correct user '#{owner_id}' in mongo."
      end
    end

    domain_namespace = domain_namespace_hash[domain_id]
    if app['domain_namespace'].nil?
      summary << "Application '#{app['name']}' for domain '#{domain_id}' does not have the denormalized domain_namespace set in mongo."
    elsif domain_namespace != app['domain_namespace'].to_s
      summary << "Application '#{app['name']}' for domain '#{domain_id}' has a denormalized domain_namespace of '#{app['domain_namespace']}' instead of the correct canonical namespace '#{domain_namespace}' in mongo."
    end

    if app['members'].present?
      # add the member ssh keys
      app['members'].each do |m|
        # we are passsing the resource as nil for now since we don't have the mongoid object 
        # and the resource is ignored for :ssh_to_gears 
        if Ability.has_permission?(m["_id"], :ssh_to_gears, Application, m["r"], nil)
          app_ssh_keys |= user_hash[m["_id"].to_s]["ssh_keys"] unless user_hash[m["_id"].to_s].nil?
        end
      end
    else
      summary << "Application '#{app['name']}' for domain '#{domain_id}' has no members in mongo."
    end

    if (current_time - creation_time) > 600
      # check for component instances without corresponding group instances and vice-versa
      gi_hash = {}
      app["group_instances"].each do |gi|
        gi_hash[gi["_id"].to_s] = false
      end

      if app['component_instances']
        app['component_instances'].each do |ci|
          if ci['group_instance_id'] and gi_hash.has_key? ci['group_instance_id'].to_s
            gi_hash[ci['group_instance_id'].to_s] = true
          else
            summary << "Application '#{app['name']}' with Id '#{app['_id']}' has missing group instance for component instance '#{ci['_id']}'."
          end
        end
        gi_hash.each do |gi_id, present|
          unless present
            summary << "Application '#{app['name']}' with Id '#{app['_id']}' has no components for group instance with Id '#{gi_id}'"
          end
        end
      else
        summary << "Application '#{app['name']}' with Id '#{app['_id']}' doesn't have any component instances"
      end
    end
  end
  app['group_instances'].each do |gi|
    gi['gears'].each do |gear|
      gear_count += 1
      datastore_hash[gear['uuid'].to_s] = [login, creation_time, gear['uid'], gear['server_identity'], app["_id"].to_s, app_ssh_keys ]

      has_gears = true
      has_dns_gear = true if gear["app_dns"]
      
      if level >= 1 and $districts_enabled
        # record all used uid values for each node to match later with the district
        gear_uid_hash[gear['server_identity']] = [] unless gear_uid_hash.has_key?(gear['server_identity'])
        gear_uid_hash[gear['server_identity']] << gear['uid'].to_i
      end
    end
  end
  
  # if there are no gears, it will be caught by the check to detect missing group_instances / gears
  if has_gears and !has_dns_gear
    summary << "Application '#{app['name']}' with Id '#{app['_id']}' has DNS gear missing."
  end 
  
  user_hash[owner_id]["domains"][domain_id] += gear_count if level >= 1 and user_hash[owner_id]

  if level >= 1 and $billing_enabled and owner_id and user_hash[owner_id] and !user_hash[owner_id]['parent_user_id']
    gi_hash = {}
    app['group_instances'].each do |gi|
      gid = gi['_id'].to_s
      gi_hash[gid] = {}
      gi_hash[gid]['premium_carts'] = []
      gi_hash[gid]['addtl_fs_gb'] = 0
    end
    # Get premium carts for the group instance
    premium_carts = get_premium_carts
    app['component_instances'].each do |ci|
      if premium_carts.include?(ci['cartridge_name'])
        gid = ci['group_instance_id'].to_s
        gi_hash[gid] = {} unless gi_hash[gid]
        gi_hash[gid]['premium_carts'] = [] unless gi_hash[gid]['premium_carts']
        gi_hash[gid]['premium_carts'] << ci['cartridge_name']
      end
    end
    # Get group instances with additional storage consumption
    user_untracked_storage_limit = 0
    if owner_id and user_hash[owner_id]["max_untracked_addtl_storage_per_gear"]
      user_untracked_storage_limit = user_hash[owner_id]["max_untracked_addtl_storage_per_gear"]
    end
    app['group_overrides'].each do |go|
      next if !go['additional_filesystem_gb'] or (go['additional_filesystem_gb'] <= user_untracked_storage_limit)
      go['components'].each do |go_components|
        found = false
        app['component_instances'].each do |ci|
          if ci['cartridge_name'] == go_components['cart']
            gid = ci['group_instance_id'].to_s
            gi_hash[gid] = {} unless gi_hash[gid]
            gi_hash[gid]['addtl_fs_gb'] = (go['additional_filesystem_gb'] - user_untracked_storage_limit)
            found = true
            break
          end
        end
        break if found
      end 
    end
    # Generate gear_id_hash
    app['group_instances'].each do |gi|
      gid = gi['_id'].to_s
      gid_info = [ app['_id'].to_s, app['name'], gi_hash[gid]['addtl_fs_gb'], gi_hash[gid]['premium_carts'] ]
      gi['gears'].each do |gear|
        gear_id = gear['_id'].to_s
        unless gear_id_hash[gear_id]
          gear_id_hash[gear_id] = gid_info
        else
          app_name = gear_id_hash[gear_id][1]
          summary << "Gear Id '#{gear['_id']}' for Application '#{app['name']}' is already taken by another Application '#{app_name}'"
        end
      end
    end
  end
end

total_time = (Time.now.to_f * 1000).to_i - start_time
puts "Time to fetch mongo data: #{total_time.to_f/1000}s"
puts "Total gears found in mongo: #{datastore_hash.length}"

if level >= 1
  user_hash.each do |owner_id, owner_hash|
    total_gears = 0
    owner_hash["domains"].each { |dom_id, domain_gear_count| total_gears += domain_gear_count }

    print "Checking consumed gear count for user #{owner_hash['login']}...\t" if verbose
    if owner_hash['consumed_gears'] != total_gears
      user_consumed_gears, app_actual_gears = check_consumed_gears(owner_id)
      if user_consumed_gears != app_actual_gears
        puts "FAIL" if verbose
        msg = "User #{owner_hash['login']} has a mismatch in consumed gears (#{user_consumed_gears}) and actual gears (#{app_actual_gears})"
        summary << msg
      elsif verbose
        puts "OK"
      end
    elsif verbose
      puts "OK"
    end
  end
end

get_all_gears_start_time = (Time.now.to_f * 1000).to_i
node_hash, gear_sender_hash = OpenShift::ApplicationContainerProxy.get_all_gears
total_time = (Time.now.to_f * 1000).to_i - get_all_gears_start_time
puts "Time to get all gears from nodes: #{total_time.to_f/1000}s"
puts "Total gears found on the nodes: #{node_hash.length}"
puts "Total nodes that responded : #{gear_sender_hash.length}"

if level >= 1
  get_all_sshkeys_start_time = (Time.now.to_f * 1000).to_i
  gear_sshkey_hash, sshkeys_sender_list = OpenShift::ApplicationContainerProxy.get_all_gears_sshkeys
  total_time = (Time.now.to_f * 1000).to_i - get_all_sshkeys_start_time
  puts "Time to get all sshkeys for all gears from nodes: #{total_time.to_f/1000}s"
  puts "Total gears found on the nodes: #{gear_sshkey_hash.length}"
  puts "Total nodes that responded : #{sshkeys_sender_list.length}"
end

# now check
non_responding_nodes = {}
puts "Checking application gears and ssh keys on corresponding nodes:" if verbose
datastore_hash.each do |gear_uuid, gear_info|
  login = gear_info[0]
  creation_time = gear_info[1]
  server_identity = gear_info[3]
  app_id = gear_info[4]
  db_sshkeys = gear_info[5]
  print "#{gear_uuid} : #{gear_uuid.class}...\t" if verbose
  if (current_time - creation_time) > 600
    if not node_hash.has_key? gear_uuid
      if gear_sender_hash.has_key? server_identity
        if datastore_has_gear?(gear_uuid, app_id, false)
          puts "FAIL" if verbose
          summary << "Gear #{gear_uuid} does not exist on any node"
        elsif verbose
          puts "OK"
        end
      elsif datastore_has_gear?(gear_uuid, app_id, false)
        non_responding_nodes[server_identity] = 0 unless non_responding_nodes.has_key? server_identity
        non_responding_nodes[server_identity] += 1
        puts "FAIL" if verbose
      elsif verbose
        puts "OK"
      end
    elsif server_identity != node_hash[gear_uuid][0]
      puts "FAIL" if verbose
      summary << "Gear #{gear_uuid} exists but node '#{node_hash[gear_uuid][0]}' does not match DB server_identity '#{server_identity}'"
    elsif verbose
      puts "OK"
    end
  elsif verbose
    puts "OK"
  end

  if level >= 1
    print "Checking ssh keys for gear: #{gear_uuid}...\t" if verbose
    if (current_time - creation_time) > 600
      if gear_sshkey_hash.has_key? gear_uuid
        gear_sshkeys_list = gear_sshkey_hash[gear_uuid].uniq.sort
        db_sshkeys_list = db_sshkeys.uniq.map! {|key| "OPENSHIFT-#{gear_uuid}-#{key}"}.sort
        if db_sshkeys_list == gear_sshkeys_list
          puts "OK" if verbose
        else
          puts "FAIL" if verbose
          
          # calculate the common ssh keys in mongo and on the node
          common_sshkeys = gear_sshkeys_list & db_sshkeys_list

          # get the unmatched ssh keys for the gear from the node
          extra_gear_sshkeys = gear_sshkeys_list - common_sshkeys
          extra_gear_sshkeys.each do |key|
            summary << "Gear '#{gear_uuid}' has key with hash '#{key.split(SSH_KEY_COMMENT_DELIMITER)[1]}' and comment '#{key.split(SSH_KEY_COMMENT_DELIMITER)[0]}' on the node but not in mongo."
          end
          
          # get the unmatched ssh keys for the gear in mongo
          extra_db_sshkeys = db_sshkeys_list - common_sshkeys
          extra_db_sshkeys.each do |key|
            remove_str = "OPENSHIFT-#{gear_uuid}-"
            summary << "Gear '#{gear_uuid}' has key with hash '#{key.split(SSH_KEY_COMMENT_DELIMITER)[1]}' and updated name '#{key.split(SSH_KEY_COMMENT_DELIMITER)[0].sub(remove_str, '')}' in mongo but not on the node."
          end
        end
      elsif verbose
        # the case where gear is not returned from mcollective will be handled by the earlier checks 
        puts "OK"
      end
    elsif verbose
      puts "OK"
    end
  end
end

# print error messages for any non-responding nodes
non_responding_nodes.each do |server_identity, gear_count|
  summary << "The node #{server_identity} expected to contain #{gear_count} gears wasn't returned from mcollective for the gear list"
end

# now check reverse
puts "Checking node gears in application database:" if verbose
node_hash.each do |gear_uuid, gear_info|
  print "#{gear_uuid}...\t" if verbose
  datastore_gear_info = datastore_hash[gear_uuid]
  if !datastore_gear_info
    if !datastore_has_gear?(gear_uuid, nil, true)
      puts "FAIL" if verbose
      summary << "Gear #{gear_uuid} exists on node #{gear_info[0]} (uid: #{gear_info[1]}) but does not exist in mongo database"
    elsif verbose
      puts "OK"
    end
  else
    puts "OK" if verbose
    if !datastore_gear_info[2].nil?
      begin
        uid = gear_info[1]
        if uid != datastore_gear_info[2].to_i
          summary << "Gear #{gear_uuid} is using uid: '#{uid}' but has reserved uid: '#{datastore_gear_info[2].to_i}'"
        end
      rescue Exception => e
        summary << "Failed to check gear: '#{gear_uuid}'s uid because of exception: #{e.message}"
      end
    end
  end
end

if level >= 1
  # check for applications without any group instances in the database
  puts "Checking for application without any group instances in the database:" if verbose
  query = {"group_instances.0" => {"$exists" => false}}
  selection = {:fields => ["name", "created_at"], :timeout => false}
  OpenShift::DataStore.find(:applications, query, selection) do |app|
    print "Application #{app['name']}/#{app['_id']}...\t" if verbose
    if (current_time - app['created_at']) > 600
      puts "FAIL" if verbose
      summary << "Application '#{app['name']}' with Id '#{app['_id']}' does not have any group instances"
    elsif verbose
      puts "OK"
    end
  end

  # check for applications without any gears in the group instance
  puts "Checking for application without any gears in the group instances in the database:" if verbose
  query = {"group_instances" => {"$elemMatch" => { "gears" => {"$size" => 0}}}}
  selection = {:fields => ["name", "group_instances._id", "group_instances.gears._id", "created_at"], :timeout => false}
  OpenShift::DataStore.find(:applications, query, selection) do |app|
    print "Application #{app['name']}/#{app['_id']}...\t" if verbose
    if (current_time - app['created_at']) > 600
      app['group_instances'].each do |gi|
        if (not gi.has_key?("gears")) or gi['gears'].length == 0 
          puts "FAIL" if verbose
          summary << "Application '#{app['name']}' with Id '#{app['_id']}' does not have any gears within group instance '#{gi['_id']}'"
        end
      end
    elsif verbose
      puts "OK"
    end
  end

  # check for applications where the gears attribute is missing from the group instance
  puts "Checking for application without the gears attribute in the group instances in the database:" if verbose
  query = {"group_instances.gears" => {"$exists" => false}}
  selection = {:fields => ["name", "group_instances._id", "created_at"], :timeout => false}
  OpenShift::DataStore.find(:applications, query, selection) do |app|
    print "Application #{app['name']}/#{app['_id']}...\t" if verbose
    if (current_time - app['created_at']) > 600
      if gis = app['group_instances']
        gis.each do |gi|
          if (not gi.has_key?("gears")) or gi['gears'].length == 0 
            puts "FAIL" if verbose
            summary << "Application '#{app['name']}' with Id '#{app['_id']}' does not have the gears attribute within group instance '#{gi['_id']}'"
          end
        end
      else
        puts "FAIL" if verbose
        summary << "Application '#{app['name']}' with Id '#{app['_id']}' does not have any group_instances"
      end
    elsif verbose
      puts "OK"
    end
  end

  # check for users with nil or empty or missing login in the database
  puts "Checking for users with nil or empty or missing login in the database:" if verbose
  query = {"$or" => [{"login" => {"$type" => 10}}, {"login" => ""}, {"login" => {"$exists" => false}}]}
  selection = {:fields => ["_id"], :timeout => false}
  OpenShift::DataStore.find(:cloud_users, query, selection) do |user|
    summary << "User with Id #{user['_id']} has a null, empty, or missing login."
  end

  if $districts_enabled
    # check for any unreserved uid in the district
    # these are uids that gears are using but are still present in the district's available_uids
    puts "Checking for unreserved UIDs in the district:" if verbose
    gear_uid_hash.each do |server_identity, uid_list|
      district_hash.each do |district_id, district_info|
        if district_info[2].include?(server_identity)
          unreserved_uids = uid_list & district_info[3]
          unreserved_uids.each do |unreserved_uid|
            # re-checking unreserved UID in the database
            print "Re-checking UID #{unreserved_uid} in district #{district_info[0]} in the database...\t" if verbose
            if not datastore_has_gear_uid?(unreserved_uid, [server_identity], false)
              # the UID is no longer being used by any gear
              puts "OK" if verbose
            elsif not district_has_available_uid?(district_id, unreserved_uid, false)
              # the UID is available in the district
              puts "OK" if verbose
            else
              puts "FAIL" if verbose
              summary << "UID '#{unreserved_uid}' is available in district '#{district_info[0]}' but used by a gear on node '#{server_identity}'"
            end
          end
          break
        end
      end
    end
  
    # check for any unused uid in the district
    # these are uids that are reserved in the district, but no gear is using
    puts "Checking for unused UIDs in the district:" if verbose 
    district_used_uids = []
    district_hash.each do |district_id, district_info|
      # collect gear uids from all nodes with server identities within this district
      district_info[2].each do |server_identity|
        district_used_uids |= (gear_uid_hash[server_identity] || [])
      end
   
      first_uuid = Rails.configuration.msg_broker[:districts][:first_uid]
      district_all_uids = []
      district_all_uids.fill(0, district_info[1]) {|i| first_uuid + i}
      district_unused_uids = district_all_uids - district_info[3] - district_used_uids 
    
      district_unused_uids.each do |unused_uid|
        # re-checking unused UID in the database
        print "Re-checking UID #{unused_uid} in district #{district_info[0]} in the database...\t" if verbose
        if datastore_has_gear_uid?(unused_uid, district_info[2], true)
          # found a gear that uses this UID
          puts "OK" if verbose
        elsif district_has_available_uid?(district_id, unused_uid, true)
          # the UID is no longer reserved in the district
          puts "OK" if verbose
        else
          puts "FAIL" if verbose
          summary << "UID '#{unused_uid}' is reserved in district '#{district_info[0]}' but not used by any gear"
        end
      end
    end  
    
    # check to see if there are multiple gears with the same uid in the same district
    puts "Checking for gears with the same UID:" if verbose 
    district_hash.each do |district_id, district_info|
      # collect gear uids from all nodes with server identities within this district
      district_used_uids = []
      server_ids = []
      district_info[2].each do |server_identity|
        server_ids << server_identity
        district_used_uids = district_used_uids.concat(gear_uid_hash[server_identity] || [])
      end
      #get all the uids that appear more than once
      reused_uids = district_used_uids.select { |e| district_used_uids.count(e) > 1 }.uniq
      if reused_uids.length > 0
        puts "FAIL" if verbose
        summary << "Below UIDs are used by multiple gears in district: '#{district_info[0]}'.  Please move one of them to another district."
        reused_uids.each do |uid|
          summary << "UID: #{uid} is used by gears:"
          datastore_hash.select{|k,v| v.include? uid and server_ids.any? {|server_id| v.include? server_id}}.each do |uuid, gear_info|
            summary << "\tGear:#{uuid} in #{gear_info[3]}"
          end
        end
      end
    end
  end
end

if level >= 1 and $billing_enabled
  usage_gear_hash = {}
  usage_storage_hash = {}
  usage_cart_hash = {}
  # Populate usage hash: Only for un-ended usage records
  selection = {:fields => ["app_name", "created_at", "gear_id", "event", "usage_type", "cart_name", "addtl_fs_gb"],
               :timeout => false}
  OpenShift::DataStore.find(:usage_records, {}, selection) do |urec|
    assign_usage_hash(urec, usage_gear_hash, usage_storage_hash, usage_cart_hash, summary, verbose)
  end
  process_usage_hash(usage_gear_hash, usage_storage_hash, usage_cart_hash)

  # Find application gears that does not have usage records and viceversa.
  usage_gear_ids = usage_gear_hash.keys
  app_gear_ids = gear_id_hash.keys
  missing_gear_ids = (usage_gear_ids - app_gear_ids) + (app_gear_ids - usage_gear_ids)
  puts "Checking gears available in applications collection but not in usage_records and viceversa: " + ((missing_gear_ids.length > 0) ? "FAIL" : "OK") if verbose

  missing_gear_ids.each do |gear_id|
    print "Re-checking for gear '#{gear_id}'...\t" if verbose
    if gear_id_hash[gear_id]
      query = {'_id' => Moped::BSON::ObjectId(gear_id_hash[gear_id][0])}
    else
      query = {'name' => usage_gear_hash[gear_id]['app_name']}
    end
    query['group_instances.gears._id'] = Moped::BSON::ObjectId(gear_id)
    app_gear_exists = Application.where(query).exists?

    query = {'gear_id' => BSON::ObjectId(gear_id), 'usage_type' => UsageRecord::USAGE_TYPES[:gear_usage]}
    selection = {:fields => ["event"], :timeout => false}
    num_end_events = num_begin_events = 0
    OpenShift::DataStore.find(:usage_records, query, selection) do |urec|
      next if !UsageRecord::EVENTS.values.include?(urec['event'])
      if urec['event'] == UsageRecord::EVENTS[:end]
        num_end_events += 1
      else
        num_begin_events += 1
      end
    end
    usage_gear_exists = (num_begin_events > num_end_events)

    if app_gear_exists and usage_gear_exists
      # Ignore, false alarm
      puts "OK" if verbose
    else
      puts "FAIL" if verbose
      if app_gear_exists
        summary << "Found application with gear Id '#{gear_id}' but could not find corresponding usage record."
      else
        summary << "Found usage record for gear Id '#{gear_id}' but could not find corresponding gear in the application."
      end
    end
  end

  # Find applications with additional storage but not in usage_records and viceversa.
  app_storage_hash = {}
  gear_id_hash.each { |gear_id, gear_info| app_storage_hash[gear_id] = gear_info if gear_info[2]!=0 }
  app_storage_gear_ids = app_storage_hash.keys
  usage_storage_gear_ids = usage_storage_hash.keys
  missing_storage_gear_ids = (usage_storage_gear_ids - app_storage_gear_ids) + (app_storage_gear_ids - usage_storage_gear_ids)
  (app_storage_gear_ids & usage_storage_gear_ids).each do |gear_id|
    missing_storage_gear_ids << gear_id if app_storage_hash[gear_id][2] != usage_storage_hash[gear_id]['addtl_fs_gb']
  end
  puts "Checking gears with additional storage in applications collection but not in usage_records and viceversa: " + ((missing_storage_gear_ids.length > 0) ? "FAIL" : "OK") if verbose

  missing_storage_gear_ids.each do |gear_id|
    print "Re-checking for gear '#{gear_id}'...\t" if verbose
    if app_storage_hash[gear_id]
      query = {'_id' => Moped::BSON::ObjectId(app_storage_hash[gear_id][0])}
    else
      query = {'name' => usage_storage_hash[gear_id]['app_name']}
    end
    query['group_instances.gears._id'] = Moped::BSON::ObjectId(gear_id)
    app = Application.where(query).first
    app_storage = 0
    group_inst = nil
    app.group_instances.each do |gi|
      gi.gears.each do |gear|
        if gear._id.to_s == gear_id
          group_inst = gi
          break
        end
      end
      break if group_inst
    end if app
    app_storage = group_inst.addtl_fs_gb if group_inst and (group_inst.addtl_fs_gb != 0)

    query = {'gear_id' => BSON::ObjectId(gear_id), 'usage_type' => UsageRecord::USAGE_TYPES[:addtl_fs_gb]}
    selection = {:fields => ["event", "addtl_fs_gb"], :timeout => false}
    num_end_events = num_begin_events = addtl_fs_gb = 0
    OpenShift::DataStore.find(:usage_records, query, selection) do |urec|
      next if !UsageRecord::EVENTS.values.include?(urec['event'])
      if urec['event'] == UsageRecord::EVENTS[:end]
        num_end_events += 1
      else
        addtl_fs_gb = urec['addtl_fs_gb']
        num_begin_events += 1
      end
    end
    usage_storage = (num_begin_events > num_end_events)

    if usage_storage and (app_storage > 0) and (app_storage == addtl_fs_gb)
      # Ignore, false alarm
      puts "OK" if verbose
    else
      puts "FAIL" if verbose
      if app_storage == 0
        summary << "Found usage record for addtl storage with gear Id '#{gear_id}' but could not find corresponding gear with addtl storage in the application."
      elsif !usage_storage
        summary << "Found addtl storage for gear Id '#{gear_id}' but could not find corresponding usage record."
      else # app_storage != addtl_fs_gb
        summary << "Found addtl storage mismatch for gear Id '#{gear_id}', #{addtl_fs_gb} in usage record vs #{app_storage} for corresponding gear in the application."
      end
    end
  end

  # Find applications with premium cartridges but not in usage_records and viceversa.
  app_cart_hash = {}
  gear_id_hash.each { |gear_id, gear_info| app_cart_hash[gear_id] = gear_info if gear_info[3] and !gear_info[3].empty? }
  app_cart_gear_ids = app_cart_hash.keys
  usage_cart_gear_ids = usage_cart_hash.keys
  missing_cart_gear_ids = (usage_cart_gear_ids - app_cart_gear_ids) + (app_cart_gear_ids - usage_cart_gear_ids)
  (usage_cart_gear_ids & app_cart_gear_ids).each do |gear_id|
    if app_cart_hash[gear_id][3].sort != usage_cart_hash[gear_id]['cart_name'].sort
      missing_cart_gear_ids << gear_id
    end
  end
  puts "Checking gears with premium cartridge in applications collection but not in usage_records and viceversa: " + ((missing_cart_gear_ids.length > 0) ? "FAIL" : "OK") if verbose

  premium_carts = get_premium_carts
  missing_cart_gear_ids.each do |gear_id|
    print "Re-checking for gear '#{gear_id}'...\t" if verbose
    if app_cart_hash[gear_id]
      query = {'_id' => BSON::ObjectId(app_cart_hash[gear_id][0])}
    else
      query = {'name' => usage_cart_hash[gear_id]['app_name']}
    end
    query['group_instances.gears._id'] = BSON::ObjectId(gear_id)
    selection = {:fields => ["component_instances.cartridge_name"], :timeout => false}
    app_carts = []
    OpenShift::DataStore.find(:applications, query, selection) do |app|
      app['component_instances'].each do |ci|
        app_carts << ci['cartridge_name'] if premium_carts.include?(ci['cartridge_name'])
      end
    end 
    app_carts.sort!

    usage_carts = []
    query = {'gear_id' => BSON::ObjectId(gear_id), 'usage_type' => UsageRecord::USAGE_TYPES[:premium_cart]}
    selection = {:fields => ["event", "cart_name"], :timeout => false}
    OpenShift::DataStore.find(:usage_records, query, selection) do |urec|
      next if !UsageRecord::EVENTS.values.include?(urec['event'])
      if urec['event'] == UsageRecord::EVENTS[:end]
        usage_carts.delete(urec['cart_name'])
      else
        usage_carts << urec['cart_name']
      end
    end
    usage_carts.sort!

    if app_carts == usage_carts
      # Ignore, false alarm
      puts "OK" if verbose
    else
      puts "FAIL" if verbose
      if (app_carts - usage_carts).length > 0
        summary << "Found premium carts #{(app_carts - usage_carts).join(',')} for gear Id '#{gear_id}' but could not find corresponding usage records."
      else
        summary << "Found usage records for premium carts #{(usage_carts - app_carts).join(',')} with gear Id '#{gear_id}' but could not find corresponding gear with premium carts in the application."
      end
    end
  end

  # Find records in usage_records mongo collection but not in usage and viceversa
  usage_records_hash = {}
  [usage_gear_hash, usage_storage_hash, usage_cart_hash].each do |usage_hash|
    usage_hash.each do |gear_id, gear_info|
      usage_records_hash[gear_id] = {'num_unended_events' => 0} unless usage_records_hash[gear_id]
      if gear_info['cart_name']
        usage_records_hash[gear_id]['num_unended_events'] += gear_info['cart_name'].length
      else
        usage_records_hash[gear_id]['num_unended_events'] += 1
      end
    end
  end
  usage_hash = {}
  selection = {:fields => ["gear_id"], :timeout => false}
  OpenShift::DataStore.find(:usage, {"end_time" => nil}, selection) do |urec|
    gear_id = urec['gear_id'].to_s
    usage_hash[gear_id] = {'num_unended_events' => 0} unless usage_hash[gear_id]
    usage_hash[gear_id]['num_unended_events'] += 1
  end
  usage_record_gear_ids = usage_records_hash.keys
  usage_gear_ids = usage_hash.keys
  missing_urecs = (usage_gear_ids - usage_record_gear_ids) + (usage_record_gear_ids - usage_gear_ids)
  (usage_record_gear_ids & usage_gear_ids).each do |gear_id|
    if usage_records_hash[gear_id]['num_unended_events'] != usage_hash[gear_id]['num_unended_events']
      missing_urecs << gear_id
    end
  end
  puts "Checking un-ended records in usage_records collection but not in usage collection and viceversa: " + ((missing_urecs.length > 0) ? "FAIL" : "OK") if verbose

  missing_urecs.each do |gear_id|
    print "Re-checking for gear '#{gear_id}'...\t" if verbose
    usage_record_begin_events = 0
    usage_record_end_events = 0
    OpenShift::DataStore.find(:usage_records, {'gear_id' => BSON::ObjectId(gear_id)}, {:fields => ["event"], :timeout => false}) do |urec|
      next if !UsageRecord::EVENTS.values.include?(urec['event'])
      if urec['event'] == UsageRecord::EVENTS[:end]
        usage_record_end_events += 1
      else
        usage_record_begin_events += 1
      end
    end
    usage_record_unended_events = usage_record_begin_events - usage_record_end_events

    usage_unended_events = Usage.where({'gear_id' => Moped::BSON::ObjectId(gear_id), 'end_time' => nil}).count

    if usage_record_unended_events == usage_unended_events
      # Ignore, false alarm
      puts "OK" if verbose
    else
      puts "FAIL" if verbose
      if usage_record_unended_events > usage_unended_events
        summary << "Found #{usage_record_unended_events-usage_unended_events} un-ended records in usage_records collection for gear Id '#{gear_id}' but could not find corresponding records in usage."
      else
        summary << "Found #{usage_unended_events-usage_record_unended_events} un-ended records in usage collection for gear Id '#{gear_id}' but could not find corresponding records in usage_records."
      end
    end
  end

  # Find user plan_id/plan_state inconsistencies in mongo vs billing provider
  billing_user_hash = {}
  user_hash.each { |k,v| billing_user_hash[k] = v if v['usage_account_id'] and (v['usage_account_id'].to_i > 0) }
  billing_api = OpenShift::BillingService.instance
  billing_api.check_inconsistencies(billing_user_hash, summary, verbose) unless billing_user_hash.empty?
end
 
puts summary.empty? ? "Success" : "Check failed.\n#{summary.join("\n")}"
if $false_positive_check_cnt >= FALSE_POSITIVE_CHECK_LIMIT
  puts "WARNING: Only checked the first #{FALSE_POSITIVE_CHECK_LIMIT} errors for false positives."
end

puts "Please refer to the oo-admin-repair tool to resolve some of these inconsistencies." unless summary.empty?

total_time = (Time.now.to_f * 1000).to_i - start_time
puts "Total time: #{total_time.to_f/1000}s"
puts "Finished at: #{Time.now}"
exit (summary.empty? ? 0 : 1)
