From 441f6ae0f02c695d62664393c4baf675f3a78a51 Mon Sep 17 00:00:00 2001
From: airled
Date: Sun, 7 Feb 2016 17:40:00 +0300
Subject: [PATCH] new server and some modifications
---
Rakefile | 2 +-
app/views/info/stats.html.erb | 27 +++++++++++----------------
config/deploy.rb | 4 ++--
config/deploy/production.rb | 2 +-
lib/parser.rb | 21 +++++++++++++--------
lib/product_comparator.rb | 4 +++-
lib/slack_message.rb | 6 +++---
7 files changed, 34 insertions(+), 32 deletions(-)
diff --git a/Rakefile b/Rakefile
index c224a30..2c8d91a 100644
--- a/Rakefile
+++ b/Rakefile
@@ -64,7 +64,7 @@ namespace :local do
desc "Stop the application runned with Unicorn."
task :stop do
- pid = File.open('/home/onliner/current/tmp/pids/unicorn.pid') { |f| f.read }
+ pid = File.open('/home/railiner/current/tmp/pids/unicorn.pid') { |f| f.read }
system("kill #{pid.strip}")
puts 'Unicorn killed.'
end
diff --git a/app/views/info/stats.html.erb b/app/views/info/stats.html.erb
index eced601..1e18fa5 100644
--- a/app/views/info/stats.html.erb
+++ b/app/views/info/stats.html.erb
@@ -1,34 +1,29 @@
Статистика
-
-
-
+
+
+
+
-
diff --git a/config/deploy.rb b/config/deploy.rb
index ce3a6e1..8b01fe3 100644
--- a/config/deploy.rb
+++ b/config/deploy.rb
@@ -7,8 +7,8 @@
# Default branch is :master
# ask :branch, `git rev-parse --abbrev-ref HEAD`.chomp
# Default deploy_to directory is /var/www/my_app_name
-set :deploy_to, '/home/onliner'
-set :tmp_dir, '/home/onliner/tmp'
+set :deploy_to, '/home/railiner'
+set :tmp_dir, '/home/railiner/tmp'
# Default value for :scm is :git
# set :scm, :git
# Default value for :format is :pretty
diff --git a/config/deploy/production.rb b/config/deploy/production.rb
index 861bd62..ac6ddf7 100644
--- a/config/deploy/production.rb
+++ b/config/deploy/production.rb
@@ -6,7 +6,7 @@
# server 'example.com', user: 'deploy', roles: %w{app db web}, my_property: :my_value
# server 'example.com', user: 'deploy', roles: %w{app web}, other_property: :other_value
# server 'db.example.com', user: 'deploy', roles: %w{db}
-server '198.20.105.55', user: 'onliner', roles: %w{app db web}, my_property: :my_value
+server '51.254.10.211', user: 'railiner', roles: %w{app db web}, my_property: :my_value
# role-based syntax
diff --git a/lib/parser.rb b/lib/parser.rb
index 5102450..d027607 100644
--- a/lib/parser.rb
+++ b/lib/parser.rb
@@ -8,6 +8,8 @@
#main script for saving groups of categories, categories of products and products in database
class Parser
+ include Slack_message
+
URL = 'http://catalog.onliner.by/'
def initialize(daemon=false, queue=false)
@@ -26,9 +28,8 @@ def queue?
def run
begin
# clear_sidekiq
- Process.daemon if daemon?
- File.open("#{File.expand_path('../../tmp/pids', __FILE__)}/parser.pid", 'w') { |f| f << Process.pid }
- Slack_message.new.send("Parser started by #{ENV['USER']}@#{`hostname`.strip} on #{Time.now}", 'warning')
+ Process.daemon && save_pid if daemon?
+ send_message("Parser started by #{ENV['USER']}@#{`hostname`.strip} on #{Time.now}", 'warning')
start_stats = stats_now
Proxies_getter.perform_async('http://xseo.in/freeproxy') if queue?
html = get_html(URL)
@@ -52,12 +53,16 @@ def run
results(start_stats, stop_stats)
rescue => exception
puts exception.message
- Slack_message.new.send("Parser exception: #{exception.message} on #{Time.now}", 'danger')
+ send_message("Parser exception: #{exception.message} on #{Time.now}", 'danger')
end
end #def
private
+ def save_pid
+ File.open("#{File.expand_path('../../tmp/pids', __FILE__)}/parser.pid", 'w') { |f| f << Process.pid }
+ end
+
def check_group(name_ru)
group_found = Group.find_by(name_ru: name_ru)
group_found.nil? ? create_group(name_ru) : group_found
@@ -68,8 +73,8 @@ def clear_sidekiq
Sidekiq.redis { |c| c.del('stat:failed') }
end
- def get_html(source)
- Nokogiri::HTML(Curl.get(source).body)
+ def get_html(url)
+ Nokogiri::HTML(Curl.get(url).body)
end
def translate_to_en(word)
@@ -102,7 +107,7 @@ def special_request(url)
http.ssl_verify_peer = false
http.headers["User-Agent"] = user_agents[rand(user_agents.size)]
end
- data.body_str
+ data.body
end
#fetch all products from all the pages of one category
@@ -145,7 +150,7 @@ def results(start_stats, stop_stats)
time_result = "Done in #{time}"
db_result = "Got: #{deltas[1]} groups, #{deltas[2]} categories, #{deltas[3]} products"
puts "#{time_result}\n#{db_result}"
- Slack_message.new.send("Parser finished in #{time}. #{db_result} on #{Time.now}", 'good')
+ send_message("Parser finished in #{time}. #{db_result} on #{Time.now}", 'good')
end
end
diff --git a/lib/product_comparator.rb b/lib/product_comparator.rb
index de02aa2..22f80b6 100644
--- a/lib/product_comparator.rb
+++ b/lib/product_comparator.rb
@@ -2,6 +2,8 @@
class Comparator
+ include Slack_message
+
def run(category, product, with_queue, page_url)
begin
#product is after JSON.parse
@@ -17,7 +19,7 @@ def run(category, product, with_queue, page_url)
rescue => exception
message = "Exception: #{exception.message} | Failed product: #{product['full_name']} | Page url: #{page_url}"
puts message
- Slack_message.new.send("#{Time.now} : #{message}", 'danger')
+ send_message("#{Time.now} : #{message}", 'danger')
raise "Comparator error. Failed to handle the product data."
end
end
diff --git a/lib/slack_message.rb b/lib/slack_message.rb
index ecaaa72..5da02be 100644
--- a/lib/slack_message.rb
+++ b/lib/slack_message.rb
@@ -1,8 +1,8 @@
-require 'curb'
+module Slack_message
-class Slack_message
+ require 'curb'
- def send(message, status)
+ def send_message(message, status)
payload = {'color' => status, 'fields' => [{'value' => message}]}.to_json
Curl.post(ENV["PARSER_HOOK"], payload)
end