Support tests on Scrapyd server with auth enabled (#66)

.circleci/config.yml

@@ -74,7 +74,7 @@ jobs:
           name: Launch Scrapyd
           command: |
             cd ~
-            printf "[scrapyd]\nusername = admin\n#password = 12345\n" > scrapyd.conf
+            printf "[scrapyd]\nusername = admin\npassword = 12345\n" > scrapyd.conf
             nohup ~/repo/venv/bin/scrapyd > ~/scrapyd.log 2>&1 &
             sleep 5
             cat ~/scrapyd.log

requirements.txt

@@ -8,3 +8,4 @@ requests>=2.21.0
 setuptools>=40.6.3
 six>=1.12.0
 SQLAlchemy>=1.2.15
+w3lib>=1.17.0

scrapydweb/__init__.py

@@ -121,6 +121,7 @@ def handle_db(app):
     db.init_app(app)  # https://flask-sqlalchemy.pocoo.org/2.3/contexts/
     db.create_all()
 
+    # https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-vii-error-handling
     @app.teardown_request
     def handle_db_session(exception):
         if exception:

scrapydweb/common.py

@@ -9,7 +9,7 @@
 from flask import Response
 import requests
 from requests.adapters import HTTPAdapter
-from requests.auth import _basic_auth_str
+from w3lib.http import basic_auth_header
 
 from .__version__ import __version__
 from .models import Metadata, db
@@ -23,7 +23,7 @@
 # https://flask.pocoo.org/snippets/category/authentication/
 def authenticate():
     """Sends a 401 response that enables basic auth"""
-    return Response("<script>alert('FAIL to login: basic auth for ScrapydWeb has been enabled');</script>",
+    return Response("<script>alert('Fail to login: basic auth for ScrapydWeb has been enabled');</script>",
                     401, {'WWW-Authenticate': 'Basic realm="ScrapydWeb Basic Auth Required"'})
 
 
@@ -50,7 +50,7 @@ def get_response_from_view(url, auth=None, data=None, as_json=False):
     # python - Flask test_client() doesn't have request.authorization with pytest
     client = app.test_client()
     if auth is not None:
-        headers = {'Authorization': _basic_auth_str(*auth)}
+        headers = {'Authorization': basic_auth_header(*auth)}
     else:
         headers = {}
     if data is not None:

scrapydweb/models.py

@@ -11,6 +11,7 @@
 db = SQLAlchemy(session_options=dict(autocommit=False, autoflush=True))
 
 
+# TODO: Database Migrations https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-iv-database
 # https://flask-sqlalchemy.pocoo.org/2.3/binds/#binds
 class Metadata(db.Model):
     __tablename__ = 'metadata'
@@ -38,6 +39,7 @@ def __repr__(self):
         return pformat(vars(self))
 
 
+# TODO: Timezone Conversions https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-xii-dates-and-times
 def create_jobs_table(server):
     class Job(db.Model):
         __tablename__ = server

scrapydweb/utils/send_email.py

@@ -56,14 +56,14 @@ def send_email(**kwargs):
         server.login(email_username, email_password)
         server.sendmail(from_addr, to_addrs, msg.as_string())
     except Exception as err:
-        logger.error("FAIL to send email: %s", subject)
+        logger.error("Fail to send email: %s", subject)
         try:
-            logger.info("FAIL reason: %s", err.args[-1].decode('utf8'))
+            logger.info("Fail reason: %s", err.args[-1].decode('utf8'))
         except:
             try:
-                logger.info("FAIL reason: %s", err.args[-1].decode('gbk'))
+                logger.info("Fail reason: %s", err.args[-1].decode('gbk'))
             except:
-                logger.info("FAIL reason: %s", err)
+                logger.info("Fail reason: %s", err)
         if to_retry:
             kwargs.update(to_retry=False, need_debug=True)
             logger.debug("Retrying...")

scrapydweb/views/files/log.py

@@ -5,7 +5,6 @@
 import json
 import os
 import re
-# from socket import gethostname
 from subprocess import Popen
 import sys
 import tarfile
@@ -335,6 +334,7 @@ def update_kwargs(self):
                                                       job_finished=self.job_finished, with_ext=self.with_ext,
                                                       ui=self.UI)
 
+    # TODO: https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-x-email-support
     def email_notice(self):
         job_data_default = ([0] * 8, [False] * 6, False, time.time())
         job_data = self.job_data_dict.setdefault(self.job_key, job_data_default)
@@ -353,7 +353,6 @@ def set_email_content_kwargs(self):
         # For compatibility with Python 2, use OrderedDict() to keep insertion order
         self.email_content_kwargs = OrderedDict()
         self.email_content_kwargs['SCRAPYD_SERVER'] = self.SCRAPYD_SERVER
-        # self.email_content_kwargs['hostname'] = gethostname()
         self.email_content_kwargs['project'] = self.kwargs['project']
         self.email_content_kwargs['spider'] = self.kwargs['spider']
         self.email_content_kwargs['job'] = self.kwargs['job']

scrapydweb/views/system/settings.py

@@ -82,7 +82,7 @@ def update_kwargs(self):
         # LogParser
         self.kwargs['ENABLE_LOGPARSER'] = self.ENABLE_LOGPARSER
         self.kwargs['logparser_version'] = self.LOGPARSER_VERSION
-        self.kwargs['logparser_settings_py_path'] = LOGPARSER_SETTINGS_PY_PATH
+        self.kwargs['logparser_settings_py_path'] = self.handle_slash(LOGPARSER_SETTINGS_PY_PATH)
         self.kwargs['BACKUP_STATS_JSON_FILE'] = self.BACKUP_STATS_JSON_FILE
 
         # Timer Tasks

setup.py

@@ -35,14 +35,14 @@
     install_requires=[
         "APScheduler >= 3.5.3",  # Aug 15, 2018
         "flask >= 1.0.2",  # May 2, 2018
-        # "Flask-APScheduler >= 1.11.0",  # Dec 1, 2018
         "flask-compress >= 1.4.0",  # Jan 5, 2017
         "Flask-SQLAlchemy >= 2.3.2",  # Oct 11, 2017
         "logparser == 0.8.1",
         "requests >= 2.21.0",  # Dec 10, 2018
         "setuptools >= 40.6.3",  # Dec 11, 2018
         "six >= 1.12.0",  # Dec 10, 2018
-        "SQLAlchemy >= 1.2.15"  # Dec 12, 2018
+        "SQLAlchemy >= 1.2.15",  # Dec 12, 2018
+        "w3lib >= 1.17.0",  # Feb 9, 2017
     ],
 
     entry_points={

tests/conftest.py

@@ -14,7 +14,7 @@
 # MUST be updated: _SCRAPYD_SERVER and _SCRAPYD_SERVER_AUTH
 custom_settings = dict(
     _SCRAPYD_SERVER='127.0.0.1:6800',
-    _SCRAPYD_SERVER_AUTH=None,  # Or ('yourusername', 'yourpassword')
+    _SCRAPYD_SERVER_AUTH=('admin', '12345'),  # Or None
 
     SCRAPYD_LOGS_DIR='',  # For LogParser, defaults to the 'logs' directory that resides in current user directory
 
@@ -29,7 +29,7 @@
     FROM_ADDR=os.environ.get('FROM_ADDR', '[email protected]'),
     TO_ADDRS=[os.environ.get('TO_ADDRS', '[email protected]')],
 
-    SMTP_SERVER_='',  # Used in tests/test_a_factory.py/test_check_email_with_ssl_false(), e.g. smtp.139.com
+    SMTP_SERVER_=os.environ.get('SMTP_SERVER_', ''),  # Used in test_check_email_with_ssl_false(), e.g. smtp.139.com
     SMTP_PORT_=25,
     SMTP_OVER_SSL_=False,
     SMTP_CONNECTION_TIMEOUT_=10,
@@ -45,6 +45,15 @@
 
 @pytest.fixture
 def app():
+    fake_server = 'scrapydweb-fake-domain.com:443'
+    SCRAPYD_SERVERS = [custom_settings['_SCRAPYD_SERVER'], fake_server]
+    if custom_settings['_SCRAPYD_SERVER_AUTH']:
+        username, password = custom_settings['_SCRAPYD_SERVER_AUTH']
+        authed_server = '%s:%s@%s' % (username, password, custom_settings['_SCRAPYD_SERVER'])
+        _SCRAPYD_SERVERS = [authed_server, fake_server]
+    else:
+        _SCRAPYD_SERVERS = SCRAPYD_SERVERS
+
     config = dict(
         TESTING=True,
         # SERVER_NAME='127.0.0.1:5000',  # https://flask.pocoo.org/docs/0.12/config/#builtin-configuration-values
@@ -55,13 +64,13 @@ def app():
         LOGPARSER_PID=0,
         POLL_PID=0,
 
-        SCRAPYD_SERVERS=[custom_settings['_SCRAPYD_SERVER'], 'not-exist:6801'],
+        SCRAPYD_SERVERS=SCRAPYD_SERVERS,
+        _SCRAPYD_SERVERS=_SCRAPYD_SERVERS,
         LOCAL_SCRAPYD_SERVER=custom_settings['_SCRAPYD_SERVER'],
         SCRAPYD_SERVERS_AUTHS=[custom_settings['_SCRAPYD_SERVER_AUTH'], ('username', 'password')],
         SCRAPYD_SERVERS_GROUPS=['', 'Scrapyd-group'],
         SCRAPY_PROJECTS_DIR=os.path.join(cst.CWD, 'data'),
 
-
         ENABLE_LOGPARSER=False,
 
         EMAIL_WORKING_DAYS=list(range(1, 8)),

tests/test_a_factory.py

@@ -40,7 +40,11 @@ def test_check_app_config(app, client):
 
     # In conftest.py: ENABLE_LOGPARSER=False
     assert not os.path.exists(app.config['STATS_JSON_PATH'])
+
+    # ['username:[email protected]:6800', ]
+    app.config['SCRAPYD_SERVERS'] = app.config['_SCRAPYD_SERVERS']
     check_app_config(app.config)
+
     strings = []
 
     assert app.config['LOGPARSER_PID'] is None
@@ -59,7 +63,11 @@ def test_check_app_config(app, client):
     # Test ENABLE_EMAIL = False
     if app.config.get('ENABLE_EMAIL', False):
         app.config['ENABLE_EMAIL'] = False
+
+        # ['username:[email protected]:6800', ]
+        app.config['SCRAPYD_SERVERS'] = app.config['_SCRAPYD_SERVERS']
         check_app_config(app.config)
+
         assert app.config['LOGPARSER_PID'] is None
         assert app.config['POLL_PID'] is None
         req(app, client, view='settings', kws=dict(node=1), ins='poll_pid: None')

tests/test_aa_logparser.py

@@ -40,7 +40,11 @@ def json_loads_from_file(path):
     assert not os.path.exists(app.config['DEMO_JSON_PATH'])
     app.config['ENABLE_LOGPARSER'] = True
     app.config['ENABLE_EMAIL'] = False
+
+    # ['username:[email protected]:6800', ]
+    app.config['SCRAPYD_SERVERS'] = app.config['_SCRAPYD_SERVERS']
     check_app_config(app.config)
+
     logparser_pid = app.config['LOGPARSER_PID']
     assert isinstance(logparser_pid, int) and logparser_pid > 0
     assert app.config['POLL_PID'] is None