Add API for sending text via Slack, Telegram, or Email (#83)

scrapydweb/__init__.py

@@ -143,10 +143,18 @@ def handle_db_session(exception):
 
 
 def handle_route(app):
- def register_view(view, endpoint, url_defaults_list):
+ def register_view(view, endpoint, url_defaults_list, with_node=True, trailing_slash=True):
  view_func = view.as_view(endpoint)
  for url, defaults in url_defaults_list:
- app.add_url_rule('/<int:node>/%s/' % url, defaults=defaults, view_func=view_func)
+ rule = '/<int:node>/%s' % url if with_node else '/%s' % url
+ if trailing_slash:
+ rule += '/'
+ if not with_node:
+ if defaults:
+ defaults['node'] = 1
+ else:
+ defaults = dict(node=1)
+ app.add_url_rule(rule, defaults=defaults, view_func=view_func)
 
  from .views.index import IndexView
  index_view = IndexView.as_view('index')
@@ -256,13 +264,32 @@ def register_view(view, endpoint, url_defaults_list):
  ('projects', dict(opt='listprojects', project=None, version_spider_job=None))
  ])
 
- from .views.files.parse import UploadLogView, UploadedLogView
+ # Parse Log
+ from .views.utilities.parse import UploadLogView, UploadedLogView
  register_view(UploadLogView, 'parse.upload', [('parse/upload', None)])
  register_view(UploadedLogView, 'parse.uploaded', [('parse/uploaded/<filename>', None)])
 
- from .views.files.parse import bp as bp_parse_source
+ from .views.utilities.parse import bp as bp_parse_source
  app.register_blueprint(bp_parse_source)
 
+ # Send text
+ from .views.utilities.send_text import SendTextView, SendTextApiView
+ register_view(SendTextView, 'sendtext', [('sendtext', None)])
+ register_view(SendTextApiView, 'sendtextapi', [
+ ('slack/<channel_chatid_subject>/<text>', dict(opt='slack')),
+ ('slack/<text>', dict(opt='slack', channel_chatid_subject=None)),
+ ('slack', dict(opt='slack', channel_chatid_subject=None, text=None)),
+ ('telegram/<channel_chatid_subject>/<text>', dict(opt='telegram')),
+ ('telegram/<text>', dict(opt='telegram', channel_chatid_subject=None)),
+ ('telegram', dict(opt='telegram', channel_chatid_subject=None, text=None)),
+ ('tg/<channel_chatid_subject>/<text>', dict(opt='tg')),
+ ('tg/<text>', dict(opt='tg', channel_chatid_subject=None)),
+ ('tg', dict(opt='tg', channel_chatid_subject=None, text=None)),
+ ('email/<channel_chatid_subject>/<text>', dict(opt='email')),
+ ('email/<text>', dict(opt='email', channel_chatid_subject=None)),
+ ('email', dict(opt='email', channel_chatid_subject=None, text=None)),
+ ], with_node=False, trailing_slash=False)
+
  # System
  from .views.system.settings import SettingsView
  register_view(SettingsView, 'settings', [('settings', None)])

scrapydweb/common.py

@@ -56,6 +56,7 @@ def get_response_from_view(url, auth=None, data=None, as_json=False):
  headers = {}
  if data is not None:
  response = client.post(url, headers=headers, data=data, content_type='multipart/form-data')
+ # response = client.post(url, headers=headers, data=data, content_type='application/json')
  else:
  response = client.get(url, headers=headers)
 

scrapydweb/default_settings.py

@@ -7,6 +7,7 @@
 DOCS: https://github.com/my8100/files/blob/master/scrapydweb/README.md
 文档：https://github.com/my8100/files/blob/master/scrapydweb/README_CN.md
 """
+import os
 
 
 ############################## QUICK SETUP start ##############################
@@ -174,51 +175,50 @@
 DAEMONSTATUS_REFRESH_INTERVAL = 10
 
 
-############################## Email Notice ###################################
-# In order to be notified (and stop or forcestop a job when triggered) in time,
-# you can reduce the value of POLL_ROUND_INTERVAL and POLL_REQUEST_INTERVAL,
-# at the cost of burdening both CPU and bandwidth of your servers.
-
-# Tip: set SCRAPYDWEB_BIND to the actual IP of your host, then you can visit ScrapydWeb
-# via the links attached in the email. (check out the "ScrapydWeb" section above)
-
-# Check out this link if you are using ECS of Alibaba Cloud and your SMTP server provides TCP port 25 only:
-# https://www.alibabacloud.com/help/doc-detail/56130.htm
-
-# The default is False, set it to True to enable email notification.
-ENABLE_EMAIL = False
-
-########## smtp settings ##########
-SMTP_SERVER = ''
-SMTP_PORT = 0
-SMTP_OVER_SSL = False
-
-# Config for https://mail.google.com using SSL
-# SMTP_SERVER = 'smtp.gmail.com'
-# SMTP_PORT = 465
-# SMTP_OVER_SSL = True
-
-# Config for https://mail.google.com
-# SMTP_SERVER = 'smtp.gmail.com'
-# SMTP_PORT = 587
-# SMTP_OVER_SSL = False
-
-# Config for https://mail.qq.com/ using SSL
-# SMTP_SERVER = 'smtp.qq.com'
-# SMTP_PORT = 465
-# SMTP_OVER_SSL = True
-
-# Config for http:https://mail.10086.cn/
-# SMTP_SERVER = 'smtp.139.com'
-# SMTP_PORT = 25
-# SMTP_OVER_SSL = False
-
-# The timeout in seconds for the connection attempt, the default is 10.
-SMTP_CONNECTION_TIMEOUT = 10
-
-########## sender & recipients ##########
-# Leave this option as '' to default to the FROM_ADDR option below; Otherwise, set it up
-# if your email service provider requires an username which is different from the FROM_ADDR option below to login.
+############################## Send Text ######################################
+########## usage in scrapy projects ##########
+# See the "Send Text" page
+
+########## slack ##########
+# How to create a slack app:
+# 1. Visit https://api.slack.com/apps and press the "Create New App" button.
+# 2. Enter your App Name (e.g. mybot)and select one of your Slack Workspaces, the press "Create App".
+# 3. Click the "OAuth & Permissions" menu in the sidebar on the left side of the page.
+# 4. Scroll down the page and find out "Select Permission Scopes" in the "Scopes" section
+# 5. Enter "send" and select "Send messages as <your-app-name>", then press "Save Changes"
+# 6. Scroll up the page and press "Install App to Workspace", then press "Install"
+# 7. Copy the "OAuth Access Token", e.g. xoxp-123-456-789-abcde
+# See https://api.slack.com/apps for more info
+
+# See step 1~7 above, e.g. 'xoxp-123-456-789-abcde'
+SLACK_TOKEN = os.environ.get('SLACK_TOKEN', '')
+# The default channel to use when sending text via slack, e.g. 'general'
+SLACK_CHANNEL = 'general'
+
+########## telegram ##########
+# How to create a telegram bot:
+# 1. Visit https://telegram.me/botfather to start a conversation with Telegram's bot that creates other bots.
+# 2. Send the /newbot command to create a new bot in a chat with BotFather.
+# 3. Follow the instructions to set up name and username (e.g. my_bot) for your bot.
+# 4. You would get a token (e.g. 123:abcde) after step 3.
+# 5. Visit telegram.me/<bot_username> (e.g. telegram.me/my_bot) and say hi to your bot to initiate a conversation.
+# 6. Visit https://api.telegram.org/bot<token-in-setp-4>/getUpdates to get the chat_id.
+# (e.g. Visit https://api.telegram.org/bot123:abcde/getUpdates
+# and you can find the chat_id in "chat":{"id":123456789,...)
+# See https://core.telegram.org/bots#6-botfather for more info
+
+# See step 1~4 above, e.g. '123:abcde'
+TELEGRAM_TOKEN = os.environ.get('TELEGRAM_TOKEN', '')
+# See step 5~6 above, e.g. 123456789
+TELEGRAM_CHAT_ID = int(os.environ.get('TELEGRAM_CHAT_ID', 0))
+
+########## email ##########
+# The default subject to use when sending text via email.
+EMAIL_SUBJECT = 'Email from #scrapydweb'
+
+########## email sender & recipients ##########
+# Leave this option as '' to default to the EMAIL_SENDER option below; Otherwise, set it up
+# if your email service provider requires an username which is different from the EMAIL_SENDER option below to login.
 # e.g. 'username'
 EMAIL_USERNAME = ''
 # As for different email service provider, you might have to get an APP password (like Gmail)
@@ -227,41 +227,72 @@
 # https://stackoverflow.com/a/27515833/10517783 How to send an email with Gmail as the provider using Python?
 # https://stackoverflow.com/a/26053352/10517783 Python smtplib proxy support
 # e.g. 'password4gmail'
-EMAIL_PASSWORD = ''
+EMAIL_PASSWORD = os.environ.get('EMAIL_PASSWORD', '')
 
 # e.g. '[email protected]'
-FROM_ADDR = ''
+EMAIL_SENDER = ''
 # e.g. ['[email protected]', ]
-TO_ADDRS = [FROM_ADDR]
+EMAIL_RECIPIENTS = [EMAIL_SENDER]
 
-########## email working time ##########
-# Monday is 1 and Sunday is 7.
-# e.g, [1, 2, 3, 4, 5, 6, 7]
-EMAIL_WORKING_DAYS = []
+########## email smtp settings ##########
+# Check out this link if you are using ECS of Alibaba Cloud and your SMTP server provides TCP port 25 only:
+# https://www.alibabacloud.com/help/doc-detail/56130.htm
+# Config for https://mail.google.com using SSL: ('smtp.gmail.com', 465, True)
+# Config for https://mail.google.com: ('smtp.gmail.com', 587, False)
+# Config for https://mail.qq.com using SSL: ('smtp.qq.com', 465, True)
+# Config for http:https://mail.10086.cn: ('smtp.139.com', 25, False)
+SMTP_SERVER = ''
+SMTP_PORT = 0
+SMTP_OVER_SSL = False
+# The timeout in seconds for the connection attempt, the default is 30.
+SMTP_CONNECTION_TIMEOUT = 30
 
-# From 0 to 23.
-# e.g. [9] + list(range(15, 18)) >>> [9, 15, 16, 17], or range(24) for 24 hours
-EMAIL_WORKING_HOURS = []
+
+############################## Monitor & Alert ################################
+# The default is False, set it to True to launch the poll subprocess to monitor your crawling jobs.
+ENABLE_MONITOR = False
 
 ########## poll interval ##########
+# Tip: In order to be notified (and stop or forcestop a job when triggered) in time,
+# you can reduce the value of POLL_ROUND_INTERVAL and POLL_REQUEST_INTERVAL,
+# at the cost of burdening both CPU and bandwidth of your servers.
+
 # Sleep N seconds before starting next round of poll, the default is 300.
 POLL_ROUND_INTERVAL = 300
-
 # Sleep N seconds between each request to the Scrapyd server while polling, the default is 10.
 POLL_REQUEST_INTERVAL = 10
 
+########## alert switcher ##########
+# Tip: Set the SCRAPYDWEB_BIND option the in "QUICK SETUP" section to the actual IP of your host,
+# then you can visit ScrapydWeb via the links attached in the alert.
+
+# The default is False, set it to True to enable alert via Slack, Telegram, or Email.
+# You have to set up your accounts in the "Send text" section above first.
+ENABLE_SLACK_ALERT = False
+ENABLE_TELEGRAM_ALERT = False
+ENABLE_EMAIL_ALERT = False
+
+########## alert working time ##########
+# Monday is 1 and Sunday is 7.
+# e.g, [1, 2, 3, 4, 5, 6, 7]
+ALERT_WORKING_DAYS = []
+
+# From 0 to 23.
+# e.g. [9] + list(range(15, 18)) >>> [9, 15, 16, 17], or range(24) for 24 hours
+ALERT_WORKING_HOURS = []
+
 ########## basic triggers ##########
-# Trigger email notice every N seconds for each running job.
+# Trigger alert every N seconds for each running job.
 # The default is 0, set it to a positive integer to enable this trigger.
 ON_JOB_RUNNING_INTERVAL = 0
 
-# Trigger email notice when a job is finished.
+# Trigger alert when a job is finished.
 # The default is False, set it to True to enable this trigger.
 ON_JOB_FINISHED = False
 
 ########## advanced triggers ##########
 # - LOG_XXX_THRESHOLD:
-# - Trigger email notice the first time reaching the threshold for a specific kind of log.
+# - Trigger alert the first time reaching the threshold for a specific kind of log.
 # - The default is 0, set it to a positive integer to enable this trigger.
 # - LOG_XXX_TRIGGER_STOP (optional):
 # - The default is False, set it to True to stop current job automatically when reaching the LOG_XXX_THRESHOLD.
@@ -273,8 +304,8 @@
 # - The SIGTERM signal would be sent twice resulting in an UNCLEAN shutdown, without the Scrapy stats dumped!
 # - The 'FORCESTOP' action would be executed if both of the 'STOP' and 'FORCESTOP' triggers are enabled.
 
-# Note that the 'STOP' action and the 'FORCESTOP' action would STILL be executed even when the current time
-# is NOT within the EMAIL_WORKING_DAYS and the EMAIL_WORKING_HOURS, though NO email would be sent.
+# Note that the 'STOP' action and the 'FORCESTOP' action would still be executed even when the current time
+# is NOT within the ALERT_WORKING_DAYS and the ALERT_WORKING_HOURS, though no alert would be sent.
 
 LOG_CRITICAL_THRESHOLD = 0
 LOG_CRITICAL_TRIGGER_STOP = False

scrapydweb/run.py

@@ -189,11 +189,11 @@ def parse_args(config):
  "for timer tasks") % SCHEDULER_STATE
  )
 
- ENABLE_EMAIL = config.get('ENABLE_EMAIL', False)
+ ENABLE_MONITOR = config.get('ENABLE_MONITOR', False)
  parser.add_argument(
- '-de', '--disable_email',
+ '-dm', '--disable_monitor',
  action='store_true',
- help="current: ENABLE_EMAIL = %s, append '--disable_email' to disable email notice" % ENABLE_EMAIL
+ help="current: ENABLE_MONITOR = %s, append '--disable_monitor' to disable monitor" % ENABLE_MONITOR
  )
 
  DEBUG = config.get('DEBUG', False)
@@ -237,8 +237,8 @@ def update_app_config(config, args):
  handle_metadata('scheduler_state', STATE_PAUSED)
  else:
  handle_metadata('scheduler_state', STATE_RUNNING)
- if args.disable_email:
- config['ENABLE_EMAIL'] = False
+ if args.disable_monitor:
+ config['ENABLE_MONITOR'] = False
  if args.debug:
  config['DEBUG'] = True
  if args.verbose:

scrapydweb/templates/base.html

@@ -7,7 +7,7 @@
  <title>{% block title %}{% endblock %} - ScrapydWeb</title>
 
  <link rel="stylesheet" type="text/css" href="{{ static_css_style }}">
- <!-- <script type="text/javascript" src="https://at.alicdn.com/t/font_842334_l99noezu13o.js"></script> -->
+ <!-- <script type="text/javascript" src="https://at.alicdn.com/t/font_842334_vujk670yra.js"></script> -->
  <script type="text/javascript" src="{{ static_js_icons_menu }}"></script>
  <!-- upload icon in deploy/parse page, right arrow icon in stats | projects page-->
  <link rel="stylesheet" type="text/css" href="{{ static_css_icon_upload_icon_right }}">
@@ -211,14 +211,6 @@ <h3>Files</h3>
  <span>Logs</span>
  </a>
  </li>
- <li>
- <a id="menu_parse" href="{{ g.url_menu_parse }}" onclick="showLoader();">
- <svg class="icon" aria-hidden="true">
- <use xlink:href="#icon-parse"></use>
- </svg>
- <span>Log Parser</span>
- </a>
- </li>
  {% if SHOW_SCRAPYD_ITEMS %}
  <li>
  <a id="menu_items" href="{{ g.url_menu_items }}" onclick="showLoader();">
@@ -231,6 +223,26 @@ <h3>Files</h3>
  {% endif %}
  </ul>
 
+ <h3>Utilities</h3>
+ <ul>
+ <li>
+ <a id="menu_sendtext" href="{{ g.url_menu_sendtext }}" onclick="showLoader();">
+ <svg class="icon" aria-hidden="true">
+ <use xlink:href="#icon-send"></use>
+ </svg>
+ <span>Send Text</span>
+ </a>
+ </li>
+ <li>
+ <a id="menu_parse" href="{{ g.url_menu_parse }}" onclick="showLoader();">
+ <svg class="icon" aria-hidden="true">
+ <use xlink:href="#icon-parse"></use>
+ </svg>
+ <span>Parse Log</span>
+ </a>
+ </li>
+ </ul>
+
  <h3>System</h3>
  <ul>
  <li>
@@ -262,20 +274,23 @@ <h3>System</h3>
  </ul>
 
  <ul id="links">
- <li id="scrapydweb_version">v{{ SCRAPYDWEB_VERSION }}</li>
+ <li id="scrapydweb_version">v{{ SCRAPYDWEB_VERSION }} DEV</li>
  <!-- <li id="pypi"> -->
  <!-- <a target="_blank" href="https://pypi.org/project/scrapydweb/" > -->
  <!-- <img src="https://img.shields.io/pypi/v/scrapydweb.svg" alt="https://pypi.org/project/scrapydweb/" /> -->
  <!-- </a> -->
  <!-- </li> -->
+ <!-- <li class="github"> -->
+ <!-- <a class="github-button" href="{{ GITHUB_URL }}/blob/master/HISTORY.md" aria-label="@my8100 on GitHub">scrapydweb</a> -->
+ <!-- </li> -->
+ <!-- <li class="github"> -->
+ <!-- <a class="github-button" href="{{ GITHUB_URL.replace('scrapydweb', 'logparser') }}/blob/master/HISTORY.md" aria-label="@my8100 on GitHub">logparser</a> -->
+ <!-- </li> -->
+ <!-- <li class="github"> -->
+ <!-- <a class="github-button" href="{{ GITHUB_URL.replace('scrapydweb', 'scrapyd-cluster-on-heroku') }}" aria-label="@my8100 on GitHub">scrapyd-cluster</a> -->
+ <!-- </li> -->
  <li class="github">
- <a class="github-button" href="{{ GITHUB_URL }}/blob/master/HISTORY.md" aria-label="@my8100 on GitHub">scrapydweb</a>
- </li>
- <li class="github">
- <a class="github-button" href="{{ GITHUB_URL.replace('scrapydweb', 'logparser') }}/blob/master/HISTORY.md" aria-label="@my8100 on GitHub">logparser</a>
- </li>
- <li class="github">
- <a class="github-button" href="{{ GITHUB_URL.replace('scrapydweb', 'scrapyd-cluster-on-heroku') }}" aria-label="@my8100 on GitHub">scrapyd-cluster</a>
+ <a class="github-button" href="{{ GITHUB_URL.replace('/scrapydweb', '') }}" aria-label="@my8100 on GitHub">GitHub</a>
  </li>
  </ul>
  </aside>