From 441f6ae0f02c695d62664393c4baf675f3a78a51 Mon Sep 17 00:00:00 2001 From: airled Date: Sun, 7 Feb 2016 17:40:00 +0300 Subject: [PATCH] new server and some modifications --- Rakefile | 2 +- app/views/info/stats.html.erb | 27 +++++++++++---------------- config/deploy.rb | 4 ++-- config/deploy/production.rb | 2 +- lib/parser.rb | 21 +++++++++++++-------- lib/product_comparator.rb | 4 +++- lib/slack_message.rb | 6 +++--- 7 files changed, 34 insertions(+), 32 deletions(-) diff --git a/Rakefile b/Rakefile index c224a30..2c8d91a 100644 --- a/Rakefile +++ b/Rakefile @@ -64,7 +64,7 @@ namespace :local do desc "Stop the application runned with Unicorn." task :stop do - pid = File.open('/home/onliner/current/tmp/pids/unicorn.pid') { |f| f.read } + pid = File.open('/home/railiner/current/tmp/pids/unicorn.pid') { |f| f.read } system("kill #{pid.strip}") puts 'Unicorn killed.' end diff --git a/app/views/info/stats.html.erb b/app/views/info/stats.html.erb index eced601..1e18fa5 100644 --- a/app/views/info/stats.html.erb +++ b/app/views/info/stats.html.erb @@ -1,34 +1,29 @@

Статистика

-

-
- +

+

+ +

- diff --git a/config/deploy.rb b/config/deploy.rb index ce3a6e1..8b01fe3 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -7,8 +7,8 @@ # Default branch is :master # ask :branch, `git rev-parse --abbrev-ref HEAD`.chomp # Default deploy_to directory is /var/www/my_app_name -set :deploy_to, '/home/onliner' -set :tmp_dir, '/home/onliner/tmp' +set :deploy_to, '/home/railiner' +set :tmp_dir, '/home/railiner/tmp' # Default value for :scm is :git # set :scm, :git # Default value for :format is :pretty diff --git a/config/deploy/production.rb b/config/deploy/production.rb index 861bd62..ac6ddf7 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -6,7 +6,7 @@ # server 'example.com', user: 'deploy', roles: %w{app db web}, my_property: :my_value # server 'example.com', user: 'deploy', roles: %w{app web}, other_property: :other_value # server 'db.example.com', user: 'deploy', roles: %w{db} -server '198.20.105.55', user: 'onliner', roles: %w{app db web}, my_property: :my_value +server '51.254.10.211', user: 'railiner', roles: %w{app db web}, my_property: :my_value # role-based syntax diff --git a/lib/parser.rb b/lib/parser.rb index 5102450..d027607 100644 --- a/lib/parser.rb +++ b/lib/parser.rb @@ -8,6 +8,8 @@ #main script for saving groups of categories, categories of products and products in database class Parser + include Slack_message + URL = 'http://catalog.onliner.by/' def initialize(daemon=false, queue=false) @@ -26,9 +28,8 @@ def queue? def run begin # clear_sidekiq - Process.daemon if daemon? - File.open("#{File.expand_path('../../tmp/pids', __FILE__)}/parser.pid", 'w') { |f| f << Process.pid } - Slack_message.new.send("Parser started by #{ENV['USER']}@#{`hostname`.strip} on #{Time.now}", 'warning') + Process.daemon && save_pid if daemon? + send_message("Parser started by #{ENV['USER']}@#{`hostname`.strip} on #{Time.now}", 'warning') start_stats = stats_now Proxies_getter.perform_async('http://xseo.in/freeproxy') if queue? html = get_html(URL) @@ -52,12 +53,16 @@ def run results(start_stats, stop_stats) rescue => exception puts exception.message - Slack_message.new.send("Parser exception: #{exception.message} on #{Time.now}", 'danger') + send_message("Parser exception: #{exception.message} on #{Time.now}", 'danger') end end #def private + def save_pid + File.open("#{File.expand_path('../../tmp/pids', __FILE__)}/parser.pid", 'w') { |f| f << Process.pid } + end + def check_group(name_ru) group_found = Group.find_by(name_ru: name_ru) group_found.nil? ? create_group(name_ru) : group_found @@ -68,8 +73,8 @@ def clear_sidekiq Sidekiq.redis { |c| c.del('stat:failed') } end - def get_html(source) - Nokogiri::HTML(Curl.get(source).body) + def get_html(url) + Nokogiri::HTML(Curl.get(url).body) end def translate_to_en(word) @@ -102,7 +107,7 @@ def special_request(url) http.ssl_verify_peer = false http.headers["User-Agent"] = user_agents[rand(user_agents.size)] end - data.body_str + data.body end #fetch all products from all the pages of one category @@ -145,7 +150,7 @@ def results(start_stats, stop_stats) time_result = "Done in #{time}" db_result = "Got: #{deltas[1]} groups, #{deltas[2]} categories, #{deltas[3]} products" puts "#{time_result}\n#{db_result}" - Slack_message.new.send("Parser finished in #{time}. #{db_result} on #{Time.now}", 'good') + send_message("Parser finished in #{time}. #{db_result} on #{Time.now}", 'good') end end diff --git a/lib/product_comparator.rb b/lib/product_comparator.rb index de02aa2..22f80b6 100644 --- a/lib/product_comparator.rb +++ b/lib/product_comparator.rb @@ -2,6 +2,8 @@ class Comparator + include Slack_message + def run(category, product, with_queue, page_url) begin #product is after JSON.parse @@ -17,7 +19,7 @@ def run(category, product, with_queue, page_url) rescue => exception message = "Exception: #{exception.message} | Failed product: #{product['full_name']} | Page url: #{page_url}" puts message - Slack_message.new.send("#{Time.now} : #{message}", 'danger') + send_message("#{Time.now} : #{message}", 'danger') raise "Comparator error. Failed to handle the product data." end end diff --git a/lib/slack_message.rb b/lib/slack_message.rb index ecaaa72..5da02be 100644 --- a/lib/slack_message.rb +++ b/lib/slack_message.rb @@ -1,8 +1,8 @@ -require 'curb' +module Slack_message -class Slack_message + require 'curb' - def send(message, status) + def send_message(message, status) payload = {'color' => status, 'fields' => [{'value' => message}]}.to_json Curl.post(ENV["PARSER_HOOK"], payload) end