diff --git a/.travis.yml b/.travis.yml index 8c5eaf13..b52cbe08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,15 @@ jobs: - TOXENV="py38" script: - tox --current-env + - language: generic + os: linux + dist: focal + name: "Code Test with Python 3.8.5 under Ubuntu 20.04 (Focal Fossa)" + env: + - PYTHON_VERSION="3.8.5" + - TOXENV="py38" + script: + - tox --current-env - language: generic os: osx osx_image: xcode12 @@ -32,6 +41,15 @@ jobs: - TOXENV="py37" script: - tox --current-env + - language: generic + os: linux + dist: focal + name: "Code Test with Python 3.8.2 under Ubuntu 20.04 (Focal Fossa)" + env: + - PYTHON_VERSION="3.8.2" + - TOXENV="py38" + script: + - tox --current-env - language: generic os: osx osx_image: xcode12 @@ -50,6 +68,14 @@ jobs: - TOXENV="py36" script: - tox --current-env + - language: generic + os: linux + dist: focal +env: + - PYTHON_VERSION="3.6.11" + - TOXENV="py36" + script: + - tox --current-env - language: generic os: osx osx_image: xcode12 @@ -70,6 +96,15 @@ jobs: - TOXENV="py38" script: - tox --current-env -c tox_run.ini + - language: generic + os: linux + dist: focal + name: "Run with Python 3.8.5 under Ubuntu 20.04 (Focal Fossa)" + env: + - PYTHON_VERSION="3.8.5" + - TOXENV="py38" + script: + - tox --current-env -c tox_run.ini - language: generic if: branch =~ ^dev$ os: osx @@ -106,6 +141,32 @@ jobs: script: - tox --current-env -c tox_run_mariadb.ini + - language: generic + os: linux + dist: focal + name: "Run with Python 3.8.5 under Ubuntu 20.04 (Focal Fossa) with MariaDB (10.5) as DB Type." + env: + - PYTHON_VERSION="3.8.5" + - TOXENV="py38" + addons: + mariadb: "10.5" + before_script: + - sudo systemctl restart mysql + - sudo mysql -u root -h localhost -e "CREATE DATABASE pyfunceble DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" + - sudo mysql -u root -h localhost -e "CREATE USER 'root'@'%' IDENTIFIED BY ''" + - sudo mysql -u root -h localhost -e "CREATE USER 'pyfunceble'@'localhost' IDENTIFIED BY 'pyfunceble';" + - sudo mysql -u root -h localhost -e "GRANT ALL PRIVILEGES ON pyfunceble.* TO 'pyfunceble'@'localhost';" + + - mkdir -p "tests_dir" + - echo 'PYFUNCEBLE_DB_CHARSET=utf8mb4' > tests_dir/.pyfunceble-env + - echo 'PYFUNCEBLE_DB_HOST="/var/run/mysqld/mysqld.sock"' >> tests_dir/.pyfunceble-env + - echo 'PYFUNCEBLE_DB_NAME="pyfunceble"' >> tests_dir/.pyfunceble-env + - echo 'PYFUNCEBLE_DB_PASSWORD="pyfunceble"' >> tests_dir/.pyfunceble-env + - echo 'PYFUNCEBLE_DB_PORT="3306"' >> tests_dir/.pyfunceble-env + - echo 'PYFUNCEBLE_DB_USERNAME="pyfunceble"' >> tests_dir/.pyfunceble-env + script: + - tox --current-env -c tox_run_mariadb.ini + - language: generic if: branch =~ ^dev|master$ stage: "Deployments" diff --git a/docs/components/custom-dns-server.rst b/docs/components/custom-dns-server.rst index 8202cabe..94c90330 100644 --- a/docs/components/custom-dns-server.rst +++ b/docs/components/custom-dns-server.rst @@ -34,8 +34,10 @@ to :: dns_server: - - "8.8.8.8" - - "8.8.8.8" + - "9.9.9.9" + - "2620:fe::fe" + - "88.198.70.38" + - "2a01:4f8:140:5021::39" Since :code:`v3.0.0` it is possible to assign a specific port to use with the diff --git a/docs/components/databases.rst b/docs/components/databases.rst index 03b2b608..b3e8856c 100644 --- a/docs/components/databases.rst +++ b/docs/components/databases.rst @@ -6,6 +6,7 @@ Why do we use "databases"? We use databases to store data while we run the tests. When globally talking about databases, we are indirectly talking about the following subsystems. +(JSON) * Autocontinue * InactiveDB @@ -26,41 +27,45 @@ Warnings around Database (self) management ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. warning:: - If you plan to delete everything and still manage to use PyFunceble in the - future, please use the :code:`--clean-all` argument. + If you plan to delete everything and still manage to use PyFunceble + in the future, please use the :code:`--clean-all` argument. Indeed, it will delete everything which is related to what we generated, except things like the whois database file/table which saves (almost) static data which can be reused in the future. Deleting, for example, the whois database file/table will just make - your test run for a much longer time if you retest subject that used to be - indexed into the whois database file/table. + your test run for a much longer time if you retest subject that used + to be indexed into the whois database file/table. Databases types ^^^^^^^^^^^^^^^ Since PyFunceble :code:`2.0.0` (equivalent of :code:`>=1.18.0.dev`), -we offer multiple database types which are (as per configuration) :code:`json` -(default), :code:`mariadb` and :code:`mysql`. +we offer multiple database types which are (as per configuration) +:code:`json` (default), :code:`mariadb` and :code:`mysql`. + +We however only offers our support time for opensource software, hence +Oracles MySql are not supported. if it works, great. However they should +be pretty good covered through SQLAlchemy Why different database types? """"""""""""""""""""""""""""" With the introduction of the multiprocessing logic, it became natural to -introduce other database format as it's a nightmare to update a JSON formatted -file. +introduce other database format as it's a nightmare to update a JSON +formatted file. In order to write or use a JSON formatted database, we have to load it and overwrite it completely. -It's great while working with a single CPU/process but as soon as we get out of -that scope it become unmanageable. +It's great while working with a single CPU/process but as soon as we get +out of that scope it become unmanageable. -How to use the :code:`mysql` or :code:`mariadb` format? +How to use the :code:`mariadb` or :code:`mysql` format? """"""""""""""""""""""""""""""""""""""""""""""""""""""" -1. Create a new user, password and database (optional) for PyFunceble to work - with. +1. Create a new user, password and database (optional) for PyFunceble to + work with. 2. Create a :code:`.pyfunceble-env` file at the root of your configuration directory. @@ -77,21 +82,21 @@ How to use the :code:`mysql` or :code:`mariadb` format? PYFUNCEBLE_DB_USERNAME=pyfunceble .. note:: - Since version :code:`2.4.3.dev` it is possible to use the UNIX socket - for the :code:`PYFUNCEBLE_DB_HOST` environment variable. + Since version :code:`2.4.3.dev` it is possible to use the UNIX + socket for the :code:`PYFUNCEBLE_DB_HOST` environment variable. The typical location for :code:`mysqld.sock` is :code:`/var/run/mysqld/mysqld.sock`. This have been done to make - 1. It easier to use the :code:`socket` in conjunction with a supported CI - environment/platform. + 1. It easier to use the :code:`socket` in conjunction with a + supported CI environment/platform. 2. Leaving more space on the IP-stack on local DB installations. - 3. The :code:`UNIX:SOCKET` is usually faster than the IP connection on - local runs. + 3. The :code:`UNIX:SOCKET` is usually faster than the IP + connection on local runs. :: @@ -102,10 +107,74 @@ How to use the :code:`mysql` or :code:`mariadb` format? PYFUNCEBLE_DB_PORT=3306 PYFUNCEBLE_DB_USERNAME=pyfunceble -4. Switch the :code:`db_type` index of your configuration file to :code:`mysql` - or :code:`mariadb`. +4. Switch the :code:`db_type` index of your configuration file to + :code:`mariadb` or :code:`mysql`. 5. Play with PyFunceble! .. note:: - If the environment variables are not found, you will be asked to prompt the - information. + If the environment variables are not found, you will be asked to + prompt the information. + +SQL Layout: +^^^^^^^^^^^ + +**DRAFT**: +^^^^^^^^^^ + +The layout and data within the Sql database and how they are used should +currently be following this patterns. + +alembic_version + - version_num The Current version of Pyfunceble in number + +pyfunceble_file + - :code:`id` Primary key, auto_increment + - :code:`created` creation date of the record + - :code:`modified` Date the record was last tested, (altered) + - :code:`path` source of the file tested. URI or File_path + - :code:`test_completed` (bool) this data is used for picking up a + interrupted (broken) test or in CI for auto-continue :code:`-c` + +pyfunceble_mined + - :code:`id` Primary key, auto_increment + - :code:`created` creation date of the record + - :code:`modified` Date the record was last tested, altered + - :code:`subject_id` key_ref to :code:`pyfunceble_status.id` + - :code:`file_id` key_ref to :code:`pyfunceble_file.id` + - :code:`mined` the full fqdns results of a :code:`--mining` response + +pyfunceble_status + - :code:`id` Primary key, auto_increment + - :code:`created` creation date of the record + - :code:`modified` Date the record was last tested, altered + - :code:`file_id` (one to many relation) to :code:`pyfunceble_file.id` + This is used to extracting where a record comes from. + - :code:`tested` Is the actual record tested in full (domain/URI) + - :code:`_status` ACTIVE/INACTIVE status from the PyFunceble test (Twice??) + - :code:`status` ACTIVE/INACTIVE status from the PyFunceble test (Twice??) + - :code:`_status_source` The technique to determine the status WHOIS/DNSLOOKUP (Twice??) + - :code:`status_source` The technique to determine the status WHOIS/DNSLOOKUP (Twice??) + - :code:`domain_syntax_validation` (*INT???) Would expect a (bool(true,false)). Here I'm in doubt: Does this mean there was performed a :code:`--syntax` test OR if it (0= failed, 1= past) syntax test? + - :code:`expiration_date` domain expiration date from a successful WHOIS response (shouldn't it be served true the whois table???) + - :code:`http_status_code` the HTTP code from a lookup, example: 200 = + succes, 404 file not found (suggested to be moved to new table see for reusable data) + - :code:`ipv4_range_syntax_validation` (*INT???) Would expect a (bool(true,false)) + - :code:`ipv4_syntax_validation` (*INT???) Would expect a (bool(true,false)) + - :code:`ipv6_range_syntax_validation` (*INT???) Would expect a (bool(true,false)) + - :code:`ipv6_syntax_validation` (*INT???) Would expect a (bool(true,false)) + - :code:`subdomain_syntax_validation` ?? but from current data set I would again expect a (bool) and not (*INT) as it is 0 OR 1 + - :code:`url_syntax_validation` (*INT???) Would expect a (bool(true,false)). Here I'm in doubt: Does this mean there was performed a :code:`--syntax` test OR if it (0= failed, 1= past) syntax test? + - :code:`is_complement` is this record from a :code:`--complement` test. (*INT???) Would expect a (bool(true,false)) + - :code:`test_completed` (*INT???) Would expect a (bool(true,false)) Have we done testing this record since last commit for test. + - :code:`tested_at` The date for last succeeded tested. + +pyfunceble_whois_record + - :code:`id` Primary key, auto_increment + - :code:`created` creation date of the record + - :code:`modified` Data the record was last tested, altered + - :code:`subject` the domain for which this record is stored + - :code:`expiration_date` The domain expiration data according to the WHOIS + - :code:`epoch` The domain expiration data according to the WHOIS. **just in EPOC format** + - :code:`state` the domain state based on `expiration_date` and/or `epoc` future/past + - :code:`record` [NULL]?? would expect a key_ref to :code:`pyfunceble_status.id` and as replacement for :code:`pyfunceble_whois_record.subject` + - :code:`server` the whois server holding the WHOIS data (Should be altered to separate table/(DB) for reusable data and gaining from db.cache and minimize I/O & DB size) diff --git a/docs/contributors.rst b/docs/contributors.rst index 56f78711..27771436 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -1,7 +1,9 @@ Contributors ============ -Thanks to those awesome peoples for their awesome and crazy idea(s), contribution(s) and or issue report which made or make `PyFunceble`_ a better tool. +Thanks to those awesome peoples for their awesome and crazy idea(s), +contribution(s) and or issue report which made or make `PyFunceble`_ a +better tool. :: @@ -46,10 +48,10 @@ Thanks to those awesome peoples for their awesome and crazy idea(s), contributio .. _@sjhgvr: https://github.com/sjhgvr .. _@ScriptTiger: https://github.com/ScriptTiger .. _@speedmann: https://github.com/speedmann -.. _@spirillen: https://github.com/spirillen +.. _@spirillen: https://www.mypdns.org/p/Spirillen/ .. _@Wally3K: https://github.com/WaLLy3K .. _@xxcriticxx: https://github.com/xxcriticxx .. _@ybreza: https://github.com/ybreza .. _@ZeroDot1: https://github.com/ZeroDot1 .. _Funceble: https://github.com/funilrys/funceble -.. _PyFunceble: https://github.com/funilrys/PyFunceble \ No newline at end of file +.. _PyFunceble: https://github.com/funilrys/PyFunceble diff --git a/docs/facts/known_issues.rst b/docs/facts/known_issues.rst index 2ffb5baa..ea41820d 100644 --- a/docs/facts/known_issues.rst +++ b/docs/facts/known_issues.rst @@ -11,8 +11,8 @@ Ubuntu 20.04.1 LTS Focal ^^^^^^^^^^^^^^^^^^^^^^^^ -In Ubunto release 20.04 they have removed a package name -:code:`libffi.so.6` and upgraded it with version :code:`libffi.so.7` +In Ubuntu release 20.04 they have replaced a package named +:code:`libffi.so.6` and upgraded it to version :code:`libffi.so.7` This means PyFunceble will trow an error like: diff --git a/docs/facts/they-use-d-it.rst b/docs/facts/they-use-d-it.rst index a5cd5491..937390b9 100644 --- a/docs/facts/they-use-d-it.rst +++ b/docs/facts/they-use-d-it.rst @@ -28,7 +28,9 @@ PyFunceble! * `Phishing.Database`_ * `polish-adblock-filters`_ * `polish-pihole-filters`_ -* `pornhosts`_ +* `porn-records`_ +* `pornhosts`_ (Is to be replaced by `porn-records`_) +* `Clefspeare13-pornhosts`_ * `Stop.Google.Analytics.Ghost.Spam.HOWTO`_ * `The-Big-List-of-Hacked-Malware-Web-Sites`_ * `Top-Attacking-IP-Addresses-Against-Wordpress-Sites`_ @@ -59,7 +61,9 @@ PyFunceble! .. _Phishing.Database : https://github.com/mitchellkrogza/Phishing.Database .. _polish-adblock-filters : https://github.com/MajkiIT/polish-ads-filter/blob/master/polish-adblock-filters/adblock.txt .. _polish-pihole-filters : https://github.com/MajkiIT/polish-ads-filter/blob/master/polish-pihole-filters/hostfile.txt +.. _Clefspeare13-pornhosts : https://github.com/Clefspeare13/pornhosts .. _pornhosts : https://github.com/Import-External-Sources/pornhosts +.. _porn-records : https://github.com/mypdns/porn-records .. _Stop.Google.Analytics.Ghost.Spam.HOWTO : https://github.com/mitchellkrogza/Stop.Google.Analytics.Ghost.Spam.HOWTO .. _The-Big-List-of-Hacked-Malware-Web-Sites : https://github.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites .. _Top-Attacking-IP-Addresses-Against-Wordpress-Sites : https://github.com/mitchellkrogza/Top-Attacking-IP-Addresses-Against-Wordpress-Sites diff --git a/docs/special-thanks.rst b/docs/special-thanks.rst index f9f65caf..2a140aa0 100644 --- a/docs/special-thanks.rst +++ b/docs/special-thanks.rst @@ -13,7 +13,8 @@ Thanks to those awesome organization(s), tool(s) and or people(s) for * Their support * Their testings reports -which helped and/or still help me build, test and or make `PyFunceble`_ a better tool. +which helped and/or still help me build, test and or make `PyFunceble`_ +a better tool. :: @@ -43,7 +44,8 @@ which helped and/or still help me build, test and or make `PyFunceble`_ a better - Saurabh Kumar - `@theskumar`_ (`python-dotenv`_) - ScriptTiger - `@ScriptTiger`_ - SMed79 - `@SMed79`_ -- spirillen - `@spirillen`_ +- spirillen - `@spirillen`_ (`mypdns`_) +- sqlalchemy - `@sqlalchemy`_ - The YAML Project - `@yaml`_ (`pyyaml`_) - `yWorks`_ - (`yEd Graph Editor`_) @@ -69,14 +71,16 @@ which helped and/or still help me build, test and or make `PyFunceble`_ a better .. _@faressoft: https://github.com/faressoft .. _@kennethreitz: https://github.com/kennethreitz .. _@mitchellkrogza: https://github.com/mitchellkrogza +.. _@mypdns: https://www.mypdns.org/ .. _@pi-hole: https://github.com/pi-hole/pi-hole .. _@PromoFaux: https://github.com/PromoFaux .. _@publicsuffix: https://github.com/publicsuffix .. _@rthalley: https://github.com/rthalley .. _@ScriptTiger: https://github.com/ScriptTiger .. _@SMed79: https://github.com/SMed79 -.. _@spirillen: https://github.com/spirillen +.. _@spirillen: https://www.mypdns.org/p/Spirillen/ +.. _@sqlalchemy: https://www.sqlalchemy.org/ .. _@tartley: https://github.com/tartley .. _@theskumar: https://github.com/theskumar .. _@yaml: https://github.com/yaml -.. _@ybreza: https://github.com/ybreza \ No newline at end of file +.. _@ybreza: https://github.com/ybreza