Skip to content

Commit

Permalink
new server and some modifications
Browse files Browse the repository at this point in the history
  • Loading branch information
airled committed Feb 7, 2016
1 parent 3aada36 commit 441f6ae
Show file tree
Hide file tree
Showing 7 changed files with 34 additions and 32 deletions.
2 changes: 1 addition & 1 deletion Rakefile
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ namespace :local do

desc "Stop the application runned with Unicorn."
task :stop do
pid = File.open('/home/onliner/current/tmp/pids/unicorn.pid') { |f| f.read }
pid = File.open('/home/railiner/current/tmp/pids/unicorn.pid') { |f| f.read }
system("kill #{pid.strip}")
puts 'Unicorn killed.'
end
Expand Down
27 changes: 11 additions & 16 deletions app/views/info/stats.html.erb
Original file line number Diff line number Diff line change
@@ -1,34 +1,29 @@
<h1>Статистика</h1>
<hr>
<input id="button" type="button" value="Обновить" disabled="disabled" autocomplete="off" onclick="getStats();" class="btn btn-primary">
<p></p>
<div id="info"></div>
<div class="progress" id="loading" style="display: none">
<div class="progress-bar progress-bar-striped active" role="progressbar" aria-valuenow="45" aria-valuemin="0" aria-valuemax="100" style="width: 100%"></div>
</div>
<p>
<div id="info"></div>
<div class="progress" id="loading" style="visibility: hidden">
<div class="progress-bar progress-bar-striped active" role="progressbar" aria-valuenow="45" aria-valuemin="0" aria-valuemax="100" style="width: 100%"></div>
</div>
</p>

<script type="text/javascript">

function getStats(){

var info = document.getElementById('info');
var loading = document.getElementById('loading');
var button = document.getElementById('button');

var info = document.getElementById('info'),
loading = document.getElementById('loading'),
button = document.getElementById('button');
var xhr = new XMLHttpRequest();
xhr.open("GET", "<%= xhr_stats_path %>", true);
xhr.onload = function () {
info.innerHTML = xhr.responseText;
loading.style.display = 'none';
loading.style.visibility = 'hidden';
button.removeAttribute("disabled");
};
xhr.send();
info.innerHTML = 'Сбор информации...';
loading.style.display = 'block';
loading.style.visibility = 'visible';
button.setAttribute('disabled', 'disabled');
}

getStats();

</script>

4 changes: 2 additions & 2 deletions config/deploy.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
# Default branch is :master
# ask :branch, `git rev-parse --abbrev-ref HEAD`.chomp
# Default deploy_to directory is /var/www/my_app_name
set :deploy_to, '/home/onliner'
set :tmp_dir, '/home/onliner/tmp'
set :deploy_to, '/home/railiner'
set :tmp_dir, '/home/railiner/tmp'
# Default value for :scm is :git
# set :scm, :git
# Default value for :format is :pretty
Expand Down
2 changes: 1 addition & 1 deletion config/deploy/production.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
# server 'example.com', user: 'deploy', roles: %w{app db web}, my_property: :my_value
# server 'example.com', user: 'deploy', roles: %w{app web}, other_property: :other_value
# server 'db.example.com', user: 'deploy', roles: %w{db}
server '198.20.105.55', user: 'onliner', roles: %w{app db web}, my_property: :my_value
server '51.254.10.211', user: 'railiner', roles: %w{app db web}, my_property: :my_value


# role-based syntax
Expand Down
21 changes: 13 additions & 8 deletions lib/parser.rb
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
#main script for saving groups of categories, categories of products and products in database
class Parser

include Slack_message

URL = 'https://catalog.onliner.by/'

def initialize(daemon=false, queue=false)
Expand All @@ -26,9 +28,8 @@ def queue?
def run
begin
# clear_sidekiq
Process.daemon if daemon?
File.open("#{File.expand_path('../../tmp/pids', __FILE__)}/parser.pid", 'w') { |f| f << Process.pid }
Slack_message.new.send("Parser started by #{ENV['USER']}@#{`hostname`.strip} on #{Time.now}", 'warning')
Process.daemon && save_pid if daemon?
send_message("Parser started by #{ENV['USER']}@#{`hostname`.strip} on #{Time.now}", 'warning')
start_stats = stats_now
Proxies_getter.perform_async('https://xseo.in/freeproxy') if queue?
html = get_html(URL)
Expand All @@ -52,12 +53,16 @@ def run
results(start_stats, stop_stats)
rescue => exception
puts exception.message
Slack_message.new.send("Parser exception: #{exception.message} on #{Time.now}", 'danger')
send_message("Parser exception: #{exception.message} on #{Time.now}", 'danger')
end
end #def

private

def save_pid
File.open("#{File.expand_path('../../tmp/pids', __FILE__)}/parser.pid", 'w') { |f| f << Process.pid }
end

def check_group(name_ru)
group_found = Group.find_by(name_ru: name_ru)
group_found.nil? ? create_group(name_ru) : group_found
Expand All @@ -68,8 +73,8 @@ def clear_sidekiq
Sidekiq.redis { |c| c.del('stat:failed') }
end

def get_html(source)
Nokogiri::HTML(Curl.get(source).body)
def get_html(url)
Nokogiri::HTML(Curl.get(url).body)
end

def translate_to_en(word)
Expand Down Expand Up @@ -102,7 +107,7 @@ def special_request(url)
http.ssl_verify_peer = false
http.headers["User-Agent"] = user_agents[rand(user_agents.size)]
end
data.body_str
data.body
end

#fetch all products from all the pages of one category
Expand Down Expand Up @@ -145,7 +150,7 @@ def results(start_stats, stop_stats)
time_result = "Done in #{time}"
db_result = "Got: #{deltas[1]} groups, #{deltas[2]} categories, #{deltas[3]} products"
puts "#{time_result}\n#{db_result}"
Slack_message.new.send("Parser finished in #{time}. #{db_result} on #{Time.now}", 'good')
send_message("Parser finished in #{time}. #{db_result} on #{Time.now}", 'good')
end

end
4 changes: 3 additions & 1 deletion lib/product_comparator.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

class Comparator

include Slack_message

def run(category, product, with_queue, page_url)
begin
#product is after JSON.parse
Expand All @@ -17,7 +19,7 @@ def run(category, product, with_queue, page_url)
rescue => exception
message = "Exception: #{exception.message} | Failed product: #{product['full_name']} | Page url: #{page_url}"
puts message
Slack_message.new.send("#{Time.now} : #{message}", 'danger')
send_message("#{Time.now} : #{message}", 'danger')
raise "Comparator error. Failed to handle the product data."
end
end
Expand Down
6 changes: 3 additions & 3 deletions lib/slack_message.rb
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
require 'curb'
module Slack_message

class Slack_message
require 'curb'

def send(message, status)
def send_message(message, status)
payload = {'color' => status, 'fields' => [{'value' => message}]}.to_json
Curl.post(ENV["PARSER_HOOK"], payload)
end
Expand Down

0 comments on commit 441f6ae

Please sign in to comment.