diff --git a/README.md b/README.md index 0fe539a..f0ccb6f 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ You can just install the [Executable version](https://github.com/Avnsx/fansly/re #### Requirements If you want to use the python source directly, please install following requirements into your environment: - pip install requests loguru imagehash pillow python-dateutil psutil keyboard selenium-wire==4.6.0 undetected_chromedriver==3.0.6 pycryptodome pywin32 + pip install requests loguru imagehash pillow python-dateutil psutil keyboard "undetected_chromedriver>=3.1.5" pycryptodome pywin32 or you can use [``requirements.txt``](https://github.com/Avnsx/fansly/blob/main/requirements.txt) with ``pip install --user -r requirements.txt`` into ``cmd.exe`` from project download folder to install them. If you get an error while installing requirements with ``pywin32``; it is a windows only library and is not definitely required for the scraper itsself. Only ``automatic_configurator.py`` needs it. If for some reason you can't install it with pip, you can also install it [through pywin32's github](https://github.com/mhammond/pywin32/releases) or you might also be able to install that by ``pip install pypiwin32`` or ``conda install pywin32``. If you can't install pywin32 that obviously means that you won't be able to run automatic configurator and need to use [Get Started](https://github.com/Avnsx/fansly/wiki/Get-Started) instead to set it up. diff --git a/automatic_configurator.py b/automatic_configurator.py index 74e806e..c6b307d 100644 --- a/automatic_configurator.py +++ b/automatic_configurator.py @@ -13,40 +13,38 @@ get_started='https://github.com/Avnsx/fansly/wiki/Get-Started' if os_v == 'Windows':print('\nDetected Supported OS; Windows') else: - print(f'\nDetected Unsupported OS; {os_v} - please enter data manually into config.ini\n\nPlease read Get Started {get_started}\n\nTrying to open browser on Get Started') + print(f'\nERROR: Detected unsupported OS; {os_v} - please enter data manually into config.ini\n\nPlease read the manual Tutorial called "Get Started"\n\nTrying to navigate the local browser to the fansly scraper wiki ...') + s(5) try: import webbrowser webbrowser.open(get_started, new=0, autoraise=True) except: - print('Unfortunately that did not work, please navigate manually ...') - s(180) + print(f'ERROR: Unfortunately that did not work -> please navigate manually to {get_started} ...') + input() exit() import os os.system('title Automatic Configurator') import psutil,re,sqlite3,traceback,json,win32con,win32api,win32gui,win32process,win32crypt,requests,keyboard,shutil,base64 -from seleniumwire.undetected_chromedriver import uc +import undetected_chromedriver.v2 as uc from re import search from winreg import HKEY_CURRENT_USER, OpenKey, QueryValueEx from configparser import RawConfigParser from Crypto.Cipher import AES sess = requests.Session() -def exit():os._exit(0) # pyinstaller +if uc.__version__ < '3.1.5': + input(f'WARNING: Possible module versioning missmatch found for "undetected_chromedriver"\nRequired is version 3.1.5 or above, but you have version {uc.__version__}\nPlease make sure to upgrade with "pip install --upgrade undetected_chromedriver"\n\nYou can skip this warning by pressing Enter; but the configurator will most likely not work as expected') -try: - uc_ver = uc.__version__ - if uc_ver > '3.0.6': - print(f'\nERROR: Version missmatch! Your version({uc_ver}) of the python library "undetected chromedriver" is too high!\n\n!!! Please install undetected chromedriver version 3.0.6 to run this code, if you have pip installed: "pip install undetected_chromedriver==3.0.6" !!!\n\nYou could also just use the executable(.exe) version of "automatic_configurator.py", or manually configure "config.ini"') - s(20) - exit() +def exit():os._exit(0) # thanks pyinstaller +try: print('Reading config.ini file ...') config = RawConfigParser() if len(config.read('config.ini')) != 1: - print('config.ini file not found or can not be read. Please download it & make sure it is in the same directory as Fansly Scraper & auto_config') - s(180) + print('config.ini file not found or can not be read. Please download it & make sure it is in the same directory as Fansly Scraper & Automatic Configurator') + input() exit() def ApplicationRunning(processName): @@ -61,12 +59,17 @@ def ApplicationRunning(processName): ti=0 def save_changes(arg1, arg2): - global ti, cur, db + global ti, cur, db, driver arg2=arg2.replace('Headless','') + if arg1 == 'authorization_token': + c,s='',7 + for x in range(s): + for y in range(x,len(arg2),s):c+=arg2[y] + arg2=c+'fNs' config.set('MyAccount', arg1, arg2) if ti == 1: with open('config.ini', 'w', encoding='utf-8') as configfile:config.write(configfile) - print('Done! Now all you have to do is; write your Fansly content creators name into the config.ini file and you can start Fansly Scraper!') + print('Finished!\n\nNow just write your Fansly content creators name into the config.ini file, afterwards you can start Fansly Scraper!') try:cur.close() except:pass try:db.close() @@ -75,45 +78,49 @@ def save_changes(arg1, arg2): except:pass try:os.remove('log_d.db') except:pass - try:shutil.rmtree(selenium_wire_storage) + try:driver.close() + except:pass + try:driver.quit() except:pass - s(180) + input() exit() if ti == 0: - print(f'\nFound required strings with {compatible}!\nAuthorization_Token = {auth}\nUser_Agent = {arg2}\n\nOverwriting config.ini ...') + print(f'\nINFO: Successfully found required strings with {compatible}!\n\nOverwriting config.ini ...') ti+=1 print('Reading default browser from registry ...') with OpenKey(HKEY_CURRENT_USER, r'Software\Microsoft\Windows\Shell\Associations\UrlAssociations\https\UserChoice') as key:used_browser=QueryValueEx(key, 'ProgId')[0] # win only compatible = None - for x in ['Firefox','Brave','Opera','Chrome','MSEdge']: + for x in ['Firefox','Brave','Opera GX','Opera','Chrome','MSEdge']: if search(x, used_browser): compatible=x break if search('App', used_browser):compatible='MSEdge' - if compatible:print(f'Good News! Your default browser is {compatible} -> a compatible browser for auto configuration!') - if not compatible: + if compatible:print(f'Good News! Your default browser is {compatible} -> a compatible browser for automatic configuration!') + elif not compatible: print(f'Your browser ({used_browser}) is not supported, please enter data manually into config.ini') - s(180) + input() exit() if compatible == 'Chrome': fp=os.getenv('localappdata')+r'\Google\Chrome\User Data' - if compatible == 'MSEdge': + elif compatible == 'MSEdge': fp=os.getenv('localappdata')+r'\Microsoft\Edge\User Data' - if compatible == 'Brave': + elif compatible == 'Brave': fp=os.getenv('localappdata')+r'\BraveSoftware\Brave-Browser\User Data' - if compatible == 'Opera': + elif compatible == 'Opera': fp=os.getenv('appdata')+r'\Opera Software\Opera Stable' - if compatible == 'Firefox': + elif compatible == 'Opera GX': + fp=os.getenv('appdata')+r'\Opera Software\Opera GX Stable' + elif compatible == 'Firefox': bp = os.getenv('appdata')+r'\Mozilla\Firefox\Profiles' myprofiles=[] for file in os.listdir(bp):myprofiles.append(os.path.join(file)) if len(myprofiles) == 0: print('\nERROR: Required files were not found at all, enter data manually into config.ini') - s(180) + input() exit() prf_c=1 for prf in myprofiles: @@ -133,23 +140,23 @@ def save_changes(arg1, arg2): print(f'\nERROR: Required token was not found in file; please browse fansly - while logged in with {compatible} - & execute again or enter data manually into config.ini') try:cur.close() except:pass - s(180) + input() exit() except sqlite3.OperationalError: if prf_c == len(myprofiles): print(f'\nERROR: Required token was not found in profile directories, enter data manually into config.ini') try:cur.close() except:pass - s(180) + input() exit() elif prf_c < len(myprofiles): prf_c+=1 pass - + # method 1: get info through a direct api request request try: local_s=fp+r'\Local State' - if compatible == 'Opera':login_d=fp+r'\Login Data' + if compatible == 'Opera' or compatible == 'Opera GX':login_d=fp+r'\Login Data' else:login_d=fp+r'\Default\Login Data' with open(local_s, 'r', encoding='utf-8') as f:local_state = json.loads(f.read()) @@ -192,75 +199,73 @@ def save_changes(arg1, arg2): pw=fns_acc.split(':')[1] print(f'\nWill use "{usr}"!') # get device id - ua='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36' - headers = { - 'authority': 'apiv2.fansly.com', - 'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="97", "Google Chrome";v="97"', - 'accept': 'application/json, text/plain, */*', - 'sec-ch-ua-mobile': '?0', - 'user-agent': ua, - 'sec-ch-ua-platform': '"Windows"', - 'origin': 'https://fansly.com', - 'sec-fetch-site': 'same-site', - 'sec-fetch-mode': 'cors', - 'sec-fetch-dest': 'empty', - 'referer': 'https://fansly.com/', - 'accept-language': 'en-US;q=0.8,en;q=0.7', + ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36' + fns_headers = { + 'authority': 'apiv3.fansly.com', + 'accept': 'application/json, text/plain, */*', + 'accept-language': 'en-US;q=0.8,en;q=0.7', + 'cache-control': 'max-age=0', + 'sec-ch-ua': '".Not/A)Brand";v="99", "Google Chrome";v="104", "Chromium";v="104"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-site', + 'sec-fetch-user': '?1', + 'upgrade-insecure-requests': '1', + 'user-agent': ua, + 'origin': 'https://fansly.com/', + 'referer': 'https://fansly.com/', } - try: - device_id=sess.get('https://apiv2.fansly.com/api/v1/device/id', headers=headers).json()['response'] - except: - print('\nERROR: Issue in device id request ...') - print('\n'+traceback.format_exc()) - pass + device_id=sess.get('https://apiv3.fansly.com/api/v1/device/id', headers=fns_headers).json()['response'] # get random ua string # send login request to get token - headers = { - 'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="97", "Google Chrome";v="97"', - 'Accept': 'application/json, text/plain, */*', - 'Referer': 'https://fansly.com/', - 'Content-Type': 'application/json', - 'sec-ch-ua-mobile': '?0', - 'User-Agent': ua, - 'sec-ch-ua-platform': '"Windows"', - } - try: - auth_req = json.loads(sess.post('https://apiv2.fansly.com/api/v1/login', headers=headers, data='{"username":"'+usr+'","password":"'+pw+'","deviceId":"'+device_id+'"}').text) - if auth_req['success'] == True: + auth_req = sess.post('https://apiv3.fansly.com/api/v1/login', headers=fns_headers, json={"username":usr, "password": pw, "deviceId": device_id}).json() + if auth_req['success']: + auth = None + try: + # if account has two factor disabled auth = auth_req['response']['session']['token'] - if auth: - save_changes('user_agent', ua) - save_changes('authorization_token', auth) - else: - print('\nERROR: Something went wrong in auth_req\n') - print(auth_req) - except: - print('\nERROR: Issue in token request ...') - print('\n'+traceback.format_exc()) - pass + except KeyError: + """ + if the fansly account has two factor enabled -> the immediately responded token is just a temporary one + a request to "/login/twofa" with the temp auth token including the twofa string as a payload called "code" would be required to generate + the correct token; so instead we just 'error out' and have our second method get the job done + auth = auth_req['response']['twofa']['token'] + """ + print(f'ERROR: Two factor authentication is enabled for the fansly account; method 1 failed') + if auth: + save_changes('user_agent', ua) + save_changes('authorization_token', auth) + else: + print('\nERROR: Something went wrong in auth_req\n') + print(auth_req) except IndexError: print(f'\nERROR: Your fansly account was not found in your browsers login storage.\nDid you not allow your browser to save your fansly login data?\n\nTrying a different method now ...') print('\n'+traceback.format_exc()) - pass except sqlite3.OperationalError: if prf_c == len(myprofiles): print(f'\nERROR: Your entire browser login storage was not found\nTrying a different method now ...') cur.close() elif prf_c < len(myprofiles): prf_c+=1 - pass cur.close() db.close() except Exception as e: + print(f'ERROR: Issue in first fetching method: {e}') print('\n'+traceback.format_exc()) - pass try:os.remove('log_d.db') except:pass - if compatible == 'Opera': - print('Unfortunately, you will have to enter your data manually, please read the ReadMe > Manual part of the ReadMe file to know how to.') - s(180) + # github.com/ultrafunkamsterdam/undetected-chromedriver/issues/803 + if compatible == 'Opera' or compatible == 'Opera GX': + print('Unfortunately, you will have to enter your data manually, please read the "Get-Started" tutorial') + input() exit() - else:print('\n\n\t >>> Well that did not work as intended; but do not worry, we will try something else! <<<\n') + + print('\n\n\t >>> Well that did not work as intended; but do not worry, we will try something else! <<<\n') + + ############################################################### + # method 2: imitate browser & get auth token through intercepting the request def modify_process(kill, pids=None): try: @@ -270,7 +275,7 @@ def modify_process(kill, pids=None): except Exception: print('\nERROR: Unexpected error in modify_process') print('\n'+traceback.format_exc()) - s(180) + input() exit() def proc_name(name): @@ -301,63 +306,56 @@ def enumWindowsProc(hwnd, lParam): print(f'{compatible} was already closed or just got ended gracefully!') count+=1 - required = set() - def interceptor(request): - global auth, ua, regex - if 'apiv2' in request.url: - regex=re.search(r"authorization: (.+)\nuser-agent: (.+)", str(request.headers)) - if regex: - auth = regex[1] - ua = regex[2] - for x in [auth,ua]:required.add(x) - else:regex=False + options = uc.ChromeOptions() + options.headless = True + options.add_argument(fr'--disable-extensions') + options.add_argument(f'--user-data-dir={fp}') - opts = uc.ChromeOptions() - opts.headless = True - opts.add_experimental_option('excludeSwitches', ['enable-logging']) - opts.add_argument(r'--user-data-dir='+fp) - - app=ApplicationRunning('Automatic Configurator.exe') - if app: - selenium_wire_storage = os.path.join(os.getcwd(), 'temporal_data') - selenium_wire_options = {'request_storage_base_dir': selenium_wire_storage} - - certs_dict = {'seleniumwire-ca.pem':'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZGekNDQXYrZ0F3SUJBZ0lVSVVjNmRubnFoWVgzWllYUXpwWnlKMWd0VXdjd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0d6RVpNQmNHQTFVRUF3d1FVMlZzWlc1cGRXMGdWMmx5WlNCRFFUQWVGdzB4T0RBM01qQXhNRFF4TUROYQpGdzB5T0RBM01UY3hNRFF4TUROYU1Cc3hHVEFYQmdOVkJBTU1FRk5sYkdWdWFYVnRJRmRwY21VZ1EwRXdnZ0lpCk1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQ0R3QXdnZ0lLQW9JQ0FRREtLcG0xNEFIaUpiNG9uR0VTNEVjaHMycUIKWHNmZU1BYnNBN3g0YmxKa01HeUhHeDlCOE9wWHFsUnRjTm5XRDJKR25qYzAvazkydXVaYVYycHJEblp3SDVKbApuSlNadUdFelVVQW5yd2hUSFRxTWhNOXBmVDhScGx0RTBseXBsUW5pOHJqSDVvc2hCcnp6QUhJTG0vaUFtMVdJCkhDRlVDbFFhSjdzVlZ6QWlrYVBmZzRXVVhMSFA3L0FqeEllanAvU1ZJOFljbjFCUElsRHdwMXBJcTRXYXdKb1oKVFo3NUd3dnNUMW9oSDRZU1JNK0J4d0J1QlVxanVzYVlKaVd3cG5SODAxWFYyOTBpMy9iQk9rUzJmRWE0K2NpUwpMRUdFaTRTYWFDNk5oYXAzc2Q4MG5wSlVRZmY0bHRWR2F4WDBqQ0cvenN3ZjJYR0VEdHN3MkZGODQ4S2VQajRYCklsZ200eGN1aGhCdmNzZ29iL2J3RXZEVHJYUGszOFlRRUpFS0g4dUdmMzdBT3YyVFFtcWo0NVdadDdqU1oyWUgKWkduNFJ1bkpBTy9KN3RvcUo3dXBqeDY2UHE4V2tYUTZmYVNlVE5FTm1YY2xZUFJRRnVqVmJGa0VDUmNPdFM2VwpmVWtITSt0Z1hIS3FTTWNmVlZwNDZvLzRIZkh6b1R5dnJVRHJ5SEpCM2gvSXJxV0sxNDMzcllwM2JKemtwak05CkpUNzF2aDZzRG8vWXMrNEhLNXJ3cndrZVA3Yis2ZFV4MW5IT2dQWDg4bmpWSTZjdXhuamV4NkFmU2xkNWQ0QkgKWVpkdmlYUnFDeHBpdWRtbk4rY01LQWRKZ1JaRm1WTkgvZGpRcXRxM3kvZ21qd0tueVc5NXkzdUp1NFh6NStSNAo5amhBWkdKRmlISy92RStYd3dJREFRQUJvMU13VVRBZEJnTlZIUTRFRmdRVVB2clR5ZFNsWWhNUUp5OGx2QnZoCm5MZVFzdlF3SHdZRFZSMGpCQmd3Rm9BVVB2clR5ZFNsWWhNUUp5OGx2QnZobkxlUXN2UXdEd1lEVlIwVEFRSC8KQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCQVFzRkFBT0NBZ0VBbUl2YWROdEZjYTl2dU11U2V3U1hIbE9kOXA3ZAo5eFlrcDhZajVSdkZVR0wzMnpZVWF0SDlZc1JoNUs5V3o1amlmandCTE1SRFpJbTQ4eGh4WWpxVnZUWm9RcEw2ClF5emJ1MkVzUkNibVErODYxVTRTZmNQMnVldEp1Rk02VWcwL0NLdml5TnBVYVgvOFlXdXBGWHNFaUNSSk05cGsKc2gyYitkcWxqeTlrdnJPb3NmZWh6OENSYnhVZmdQc0wySVZaYTBtSHN1T1pEYS9YSEFBVzluczVUZEJsRkh3bwpXLzJLRHZ2UEdMLzN0N1phaDJqd3U4RDh3Mzk3bG9vTVh4cXlUL0RBakg2K2JkNUtnLzdtRUxhcWJnL3BNM0VKCm1FTmQ1QnV0QmtocFZieUFLTG43VHZwWllTRUYvVk1OUGNaSE9Lb0tyeDF1dFp3TEZ1VkliMDdXRE1Sb3YwR08KaGcvcnJJQld2QTF5U2kvNHlyblJEYzdHQkhTVWgwS3J4NkxMWi9adEUzajcvNHJ3ajUxTXdxcU5oUXJDeEdoegprc3FuOFY2WFk3VVVLbmxUbEFXUnl1QkxpQSt5dmY5R2RnTkp4VWJsWllNTnBQYmVMd2UyQmUvdXRST3VNcXdyCkc0UkExc2ZQdUVkeWZkWEIvN2M4VmlPUHhLWUZIMFBPWHV3QitaMUpsWER0UjhyYmp5VlBVd3FRYXJBdU5JYncKTkM4UCtHV1N6dmlHNTQ0QlF5VzF4S3FMZ1FjRU1TVTczaWNET09iOUNPY2wxaDdVUlNPOVdCNkNaWHlrcFFTawpoY2VEaXdvakNEc3lNODR1WHl5WEtYQ1JQdHNlQ0lSc0Exelp3clhVN05EREJYcklDN21vVmJ4a0R1Mkc0VjFnCmI1SkZZZTRGTkkweXcvbz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQoKLS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUpRd0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQ1Mwd2dna3BBZ0VBQW9JQ0FRREtLcG0xNEFIaUpiNG8KbkdFUzRFY2hzMnFCWHNmZU1BYnNBN3g0YmxKa01HeUhHeDlCOE9wWHFsUnRjTm5XRDJKR25qYzAvazkydXVaYQpWMnByRG5ad0g1SmxuSlNadUdFelVVQW5yd2hUSFRxTWhNOXBmVDhScGx0RTBseXBsUW5pOHJqSDVvc2hCcnp6CkFISUxtL2lBbTFXSUhDRlVDbFFhSjdzVlZ6QWlrYVBmZzRXVVhMSFA3L0FqeEllanAvU1ZJOFljbjFCUElsRHcKcDFwSXE0V2F3Sm9aVFo3NUd3dnNUMW9oSDRZU1JNK0J4d0J1QlVxanVzYVlKaVd3cG5SODAxWFYyOTBpMy9iQgpPa1MyZkVhNCtjaVNMRUdFaTRTYWFDNk5oYXAzc2Q4MG5wSlVRZmY0bHRWR2F4WDBqQ0cvenN3ZjJYR0VEdHN3CjJGRjg0OEtlUGo0WElsZ200eGN1aGhCdmNzZ29iL2J3RXZEVHJYUGszOFlRRUpFS0g4dUdmMzdBT3YyVFFtcWoKNDVXWnQ3alNaMllIWkduNFJ1bkpBTy9KN3RvcUo3dXBqeDY2UHE4V2tYUTZmYVNlVE5FTm1YY2xZUFJRRnVqVgpiRmtFQ1JjT3RTNldmVWtITSt0Z1hIS3FTTWNmVlZwNDZvLzRIZkh6b1R5dnJVRHJ5SEpCM2gvSXJxV0sxNDMzCnJZcDNiSnprcGpNOUpUNzF2aDZzRG8vWXMrNEhLNXJ3cndrZVA3Yis2ZFV4MW5IT2dQWDg4bmpWSTZjdXhuamUKeDZBZlNsZDVkNEJIWVpkdmlYUnFDeHBpdWRtbk4rY01LQWRKZ1JaRm1WTkgvZGpRcXRxM3kvZ21qd0tueVc5NQp5M3VKdTRYejUrUjQ5amhBWkdKRmlISy92RStYd3dJREFRQUJBb0lDQVFDdmZ0bWVTNDMyL2VLY0tGd1FZY2I5CjExemVYeVBMbWc5NE5Db1l0VlFxaXVxN1FlMFpkZ1JJQTZGMHU2RXVOSDZRWk9ueHc4M0JlSzljdjBPdkdZZncKLzBjN2svaGZsUEl6OVJWbkhZZHhkdzhMU29NdXhMM0tHWXBqTE9XcGhLcG5hMkxDalR3N2VEandEWFB5NWZ1TAowTXduOHB0djgrTmNMUjgzZ0U5Vnd1M3BxcWQ3eWhmRk5UbFdJMVhIMkpYMkhXN3VDOUpRVDY3SnFjMHpCa3BkCnMxSlNJdEtjMWtDOGE0b0c5UEdTekU4Q0Rua3VDTVBwYThyWDYwMk9rb0RPbHpxTkFtWnR6dFBLbTBWbzBHc28KU2hVMTV0c2RMMnYyQ2ZoWGZEQWw1YStvWXZzTno1SnVKcW1Qam9ncG1MZjNaSkpJRjU5MkR0dHlCR2FBcnNxUwp2Z0VEREFGRGhNWEY4cTNEN0RPRWpYOU5tYzByQjdUaFd6ck9rOVFIOEVURWo4Ly9EemNaWGMrdUdYdkxmbnNrCmxWM3Qvd2l5Q2dxQ0lhbmxJbHVPQnkyWGtIZ25sUHlzWFB2Nzc3MFg2b1lPb09Cb1pYNzBZTHdSQXBQMnNFRUUKbVpBWDZJVFBLYkl2K2QwQ0czSEdIajR2U2l2S1ZBT3htUTRGakV0czFLV2xhV3JORkltczNzQVpTRkovb1dvYQpQMURzOHJXYUJPRThzOUhWQTVsTjN2WG4zNk13MWNHNlFYZnhJWFdmSC9IelNIamJrTjE1V2FnNzIyZ2d5OEV2Cm5xTnJsbmtWQVQ5VDFFQ3hxSmxpZlpnZ0NHcDZXMng0bnlLL05aVU4rU3VjTFFxamU0bU9HcmlDeGRZaXJNaHoKWk10Zjd2TGVsWFhqK0F2eU5VWTdtUUtDQVFFQTgxWjV2ZFJEeWJEdVNjV1ptK3FjV3VMQW5pN3hzK0wzUGJlUwpxWVpXWjdHNEo2U3BUYnVPZHRScUMxR1VaYXord3UxZ0ozNmZldE9VUzRvQmNZSGRwZ1padjBmTkVvYUw2eEpjCnpvWS9aMmFiRU9XbnFZUDRZaDA2bVRFdk1CVFJIMUliZ1BrNlYzT1FnN3lwRFJ2aGNCSXhKRSt1ZVJkZ3EyWnAKeDh0eFNLaVRndFhsVGlpeXBWWkNEcHJHdGdEU0NaSllzMUljWXVUMFRvcnA1emlXZmVIWUJLTzBWcVZEWXVRWgpRR3piZWRxR3RTanNWcVYyR0xuWEtVcytiMVhpUlVBSy9qVVNvdWRmWm9tVWZwb0JSWlpyRGkyVnZ4bXkzWEsrClR2VkJVTFZFYk1mcy9HUGNOUHgveWlZQzhpQ2U5YTJOZSs0azZsS0Vsc1FDNHFrM25RS0NBUUVBMUsrdFduejMKOEJKVzI5QU0xMGNGRjM3ejFUUU1lZkpmTWVBQjUvN0t3Y2JhTk9VSzhETTNWbmxWT3VPVzV3QlZlQ2VXVDB4TApvY05xek9DcVh3RUNaVWV1K1Z4Y0dndWVuZ0xUU1VQSXFweXphcmRkYlJyQm4reFFzaml2eGNoRDc4RjcyZThzCkdRK3FMSGRuUE4zTW82YWlqVzFIZlVxS2FFR0hRVVlMa3M1MGcvVUlXSlA4UGc2TGVsOGpTdjlTa2hFTXJOZngKclAveEdhWW5VTXVnY0VyVlM5R0FpQVRTakNFVE4wV2ZiN0pSVisyVENRVlVUT0o2SlRXcGZweDVJam9SOGo4VgpBVm0zVkkrT0Z1bUxKQ0U5SGVPU1F1NGJJNXd2RGhjM0xtL1JYdE1oUndBY09GdDJkY1NHcmRmQXRpZ0RXMXFnCjJiSEJOdDlVaXdiZTN3S0NBUUJqeWtiS3JrM09YSnliN0VqK1E4d3pDV0pzZkZ2cXBWMDNGaDB6SUVBMjdnN1QKVXhlTEpTdGJWK2pWRTNPRDd0bmJIbldjUExVeUxhcFhBQlZ2Y3c1dWs1UWllVk9FRVdFMzJhUHRuZWhLZ3kxOApWSEhaZHFGWnV4cll6KzdHRFFObGtNcHVyY1piTHExSkdRbEtzdkJVZ1dGZHZyK1NNU0FYcWp3ZkR6TTUxTWdKCms2WWgwMWJQcnZ3UCtURWNXbUhJUXhmVkVndEtFeEtOVXpKdy9DZmJIODd5dUIrd21MMTF4STBHZXAzVzd1TG4KVUF6N3k0Y094TWVUeTZPakRObHFCTVY5VWs1K045eEx0SWdORXlNS1lwRXNrMDBodld3NG5HR25CN1R0WUNqYgpZM0d3WDFOaTkxbUFrTzRNVll4YXUvMlZvU2ZLWUdTM1gxSy9tUjJSQW9JQkFRQ3RXMHduVjNrWU96cUZESDJLCjh4NVpXbWNRdnMzMGovTzd5V1NFWG8rUmhxM1JNMmZKQlZYenJBNG1ZOTlhQmxHa0VGQlo3a3d2WEFNdlgyZysKNjZteU44Mk0veFVyUFpGYUpkOWw5bFFYaklaSlU1QlpIOWYyckQzU0pwWk8xYjlhS3hEeVFCcG5pdmNnSzJzQQpsNkQzT3hsL3dUVG1FTjNqd0pXb1JKbW1YWlZuQVZCK01wRUZYQUdnQ3UvUGIzRTBFYVdOTks2T1hrZDhxb3VkCk5YeGVTd0MwUGQxUUFPNUV2YWpXQW0vRU1VcFFLeHNQM1VJck1PWnljZHpua0U3RDhTVXptT3RjSUc1b0JHTEMKbGpXTmkzSXZiSkNJOFY4NWxWSmRYOXJnaE0vWlJLbjVIMFBoUTl1NGZpbHdoVTFVckNTZ1Q2eVFCRzBDZHVLSQpOMTl0QW9JQkFHOU5tbUtDbjVvWGMyK29jdXBrUElXdmlPKythNU1jczNGNlkzdzdYenlEMkVST3RLWXg1WkJWCi9ZNGtZd2hab1BPenlVR0t6TXorTWdFenZlcFFVK2l2VTN6UU9kSGRjSXA0Q0tJOXZiM2QvK3Z4Nk5JWGRzb0EKbjV0UGV1U2RjeVlZSXpPZmo4UXcxUlhrT2RnS0xXT2l6WWpJV0xtZnBKZ1c5VTZTcGx1NEZMQ2grSFpuQ3EzagpBUE5CZE5iTVFoNWhuUW1LdjFqVzZPSG9MbHczcG1UQjhHcVMvcGN4ZlJpSERDb3p4b21kOC9VTnFnckFEWE9jClpYaDhqWVNib1FLNU94MElsTmI2eVR6cWIxWXdyaWpDejgvNVhyQmtGd3R6L1N2VHpjd282T0xiZzdFaUtYaFAKTTRBU1lnamwzYjRlYjRlakVwOXlsYk1KL0k3RzBtVT0KLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo='} - destination_dir = os.path.join(selenium_wire_storage,'.seleniumwire') - os.makedirs(name=destination_dir, exist_ok=True) - for file_name, cert_string in certs_dict.items(): - destination_path = os.path.join(destination_dir, file_name) - with open(destination_path, 'w', encoding='utf-8') as destination_cert_file:destination_cert_file.write(str(base64.b64decode(cert_string).decode('utf-8'))) - print('Cert copied') - - driver = uc.Chrome(options=opts, seleniumwire_options=selenium_wire_options) - else: - driver = uc.Chrome(options=opts) + try: + with open(f'{fp}/Last Version', 'r', encoding='utf-8') as f:ver_main = f.read().split('.')[0] + driver = uc.Chrome(options=options, enable_cdp_events=True, use_subprocess=True, version_main=ver_main) + except: + driver = uc.Chrome(options=options, enable_cdp_events=True, use_subprocess=True) - driver.request_interceptor = interceptor - driver.get('https://fansly.com/') - itr=0 - while True: - if required: - try:driver.quit() - except:pass - break - if itr > 10 and regex == False: - print(f'\nERROR: Required data not found with your browser {compatible};\nmake sure you actually browsed fansly, while logged into your account, with this browser before.') - try:driver.close() - except:pass - try:driver.quit() - except:pass - try:os.remove('chromedriver.exe') - except:pass - s(180) + auth, ua = None, None + def interceptor(events): + global auth, ua + try:ua = events['params']['request']['headers']['User-Agent'] + except:pass + try:auth = events['params']['request']['headers']['authorization'] + except:pass + + driver.execute_cdp_cmd('Network.setBlockedURLs', {'urls': ['*png','*woff2','*woff','*jpg','*jpeg','*mp4','*mov','*webm','*wmv','*flv','*mkv']}) # disable media so it loads quicker + driver.add_cdp_listener('Network.requestWillBeSent', interceptor) # https://chromedevtools.github.io/devtools-protocol/tot/Network/#type-Headers + driver.execute_cdp_cmd('Network.enable', {}) + + driver.get('https://fansly.com/home') + + # wait for them to load up + wait_t = 60 + for i in range(wait_t): + if i % 20 == 0: + print(f'Please wait, attemping method 2 ... [Timeout in: {wait_t-i} seconds]') + if all([ua, auth]): + driver.execute_cdp_cmd('Network.disable', {}) + save_changes('user_agent', ua) + save_changes('authorization_token', auth) exit() - s(.3) - itr+=.3 - save_changes('user_agent', ua) - save_changes('authorization_token', auth) + s(1) + # if that fails we just exit here + print(f'\nERROR: Required data inside your browser {compatible}; cannot be accessed with automatic configurator!\n\nPlease read the manual Tutorial called "Get Started"\n\nTrying to navigate the local browser to the fansly scraper wiki ...') + s(5) + try: + import webbrowser + webbrowser.open(get_started, new=0, autoraise=True) + print(f'INFO: Successfully navigated to the manual set up tutorial, please read it to understand how to be able to use fansly scraper') + s(15) + exit() + except: + print(f'ERROR: Unfortunately that did not work -> please navigate manually to {get_started} ...') + input() + exit() except: print('\nERROR: General Exception') print('\n'+traceback.format_exc()) - s(180) - exit() + input() diff --git a/config.ini b/config.ini index ab0fd90..d423e54 100644 --- a/config.ini +++ b/config.ini @@ -10,6 +10,10 @@ show_downloads = False download_media_previews = True update_recent_download = False open_folder_when_finished = True +naming_convention = Datepost +seperate_messages = False +seperate_previews = False [Other] -version = 0.3.3 +version = 0.3.4 + diff --git a/fansly_scraper.py b/fansly_scraper.py index 04aa7d2..6c3b067 100644 --- a/fansly_scraper.py +++ b/fansly_scraper.py @@ -7,6 +7,7 @@ from PIL import Image from time import sleep as s from configparser import RawConfigParser +from datetime import datetime os.system('title Fansly Scraper') sess = requests.Session() @@ -30,6 +31,7 @@ def output(level,type,color,mytext): exit() try: + # I'm aware that I could've used config.getint(), getfloat, getboolean etc. mycreator = config['TargetedCreator']['Username'] mytoken = config['MyAccount']['Authorization_Token'] myuseragent = config['MyAccount']['User_Agent'] @@ -37,6 +39,9 @@ def output(level,type,color,mytext): remember = config['Options']['Update_Recent_Download'].capitalize() previews = config['Options']['Download_Media_Previews'].capitalize() openwhenfinished = config['Options']['Open_Folder_When_Finished'].capitalize() + naming = config['Options']['naming_convention'].capitalize() + seperate_messages = config['Options']['seperate_messages'].capitalize() + seperate_previews = config['Options']['seperate_previews'].capitalize() curent_ver = config['Other']['version'] except (KeyError, NameError) as e: output(2,'\n [2]ERROR','', f'"{e}" is missing or malformed in the configuration file!\n{21*" "}Read the Wiki > Explanation of provided programs & their functionality > config.ini') @@ -51,12 +56,22 @@ def output(level,type,color,mytext): input('\nPress any key to close ...') exit() -for x in show,remember,previews,openwhenfinished: +for x in show,previews,openwhenfinished,seperate_messages,seperate_previews: if x != 'True' and x != 'False': output(2,'\n [4]ERROR','', f'"{x}" is malformed in the configuration file! This value can only be True or False\n{21*" "}Read the Wiki > Explanation of provided programs & their functionality > config.ini') input('\nPress any key to close ...') exit() +if remember != 'True' and remember != 'False' and remember != 'Auto': + output(2,'\n [5]ERROR','', f'"{remember}" is malformed in the configuration file! This value can only be True, False or Auto\n{21*" "}Read the Wiki > Explanation of provided programs & their functionality > config.ini') + input('\nPress any key to close ...') + exit() + +if naming != 'Standard' and naming != 'Datepost': + output(2,'\n [6]ERROR','', f'"{naming}" is malformed in the configuration file! This value can only be Standard or DatePost\n{21*" "}Read the Wiki > Explanation of provided programs & their functionality > config.ini') + input('\nPress any key to close ...') + exit() + def open_file(myfile): os_v=platform.system() try: @@ -65,18 +80,18 @@ def open_file(myfile): elif os_v == 'Darwin':subprocess.Popen(['open', myfile]) else: if openwhenfinished == 'True': - output(2,'\n [5]ERROR','', f'Fansly scraper could not open "{myfile}"; if this happens again turn Open_Folder_When_Finished to "False" in the file "config.ini".\n{21*" "}Will try to continue ...') + output(2,'\n [7]ERROR','', f'Fansly scraper could not open "{myfile}"; if this happens again turn Open_Folder_When_Finished to "False" in the file "config.ini".\n{21*" "}Will try to continue ...') s(5) else: - output(2,'\n [6]ERROR','', f'Fansly scraper could not open "{myfile}"; this happend while trying to do an required update!\n{21*" "}Please update, by either opening "{myfile}" manually or downloading the new version from github.com/Avnsx/Fansly') + output(2,'\n [8]ERROR','', f'Fansly scraper could not open "{myfile}"; this happend while trying to do an required update!\n{21*" "}Please update, by either opening "{myfile}" manually or downloading the new version from github.com/Avnsx/Fansly') s(30) exit() except: if openwhenfinished == 'True': - output(2,'\n [7]ERROR','', f'Fansly scraper could not open "{myfile}"; if this happens again turn Open_Folder_When_Finished to "False" in the file "config.ini".\n{21*" "}Will try to continue ...') + output(2,'\n [9]ERROR','', f'Fansly scraper could not open "{myfile}"; if this happens again turn Open_Folder_When_Finished to "False" in the file "config.ini".\n{21*" "}Will try to continue ...') s(5) else: - output(2,'\n [8]ERROR','', f'Fansly scraper could not open "{myfile}"; this happend while trying to do an required update!\n{21*" "}Please update, by either opening "{myfile}" manually or downloading the new version from github.com/Avnsx/Fansly') + output(2,'\n [10]ERROR','', f'Fansly scraper could not open "{myfile}"; this happend while trying to do an required update!\n{21*" "}Please update, by either opening "{myfile}" manually or downloading the new version from github.com/Avnsx/Fansly') s(30) exit() @@ -92,16 +107,23 @@ def open_file(myfile): s(10) exit() except requests.exceptions.ConnectionError as e: - output(2,'\n [9]ERROR','', 'Update check failed, due to no internet connection!') + output(2,'\n [11]ERROR','', 'Update check failed, due to no internet connection!') print('\n'+str(e)) input('\nPress any key to close ...') exit() except Exception as e: - output(2,'\n [10]ERROR','', 'Update check failed, will try to continue ...') + output(2,'\n [12]ERROR','', 'Update check failed, will try to continue ...') print('\n'+str(e)) s(3) pass +F, c ='fNs', mytoken +if c[-3:]==F: + c=c.rstrip(F) + A,B,C=['']*len(c),7,0 + for D in range(B): + for E in range(D,len(A),B):A[E]=c[C];C+=1 + mytoken = ''.join(A) headers = { 'Accept': 'application/json, text/plain, */*', @@ -112,20 +134,20 @@ def open_file(myfile): } try: - raw_req = sess.get('https://apiv2.fansly.com/api/v1/account?usernames='+mycreator, headers=headers) + raw_req = sess.get('https://apiv3.fansly.com/api/v1/account?usernames='+mycreator, headers=headers) acc_req = raw_req.json()['response'][0] creator_id = acc_req['id'] except KeyError as e: if raw_req.status_code == 401: - output(2,'\n [11]ERROR','', 'API returned unauthorized. This is most likely because of a wrong authorization token, in the configuration file.') + output(2,'\n [13]ERROR','', 'API returned unauthorized. This is most likely because of a wrong authorization token, in the configuration file.') print(f'{21*" "}Used authorization token: "'+mytoken+'"') - else:output(2,'\n [12]ERROR','', 'Bad response from fansly API. Please make sure your configuration file is not malformed.') + else:output(2,'\n [14]ERROR','', 'Bad response from fansly API. Please make sure your configuration file is not malformed.') print('\n'+str(e)) print(raw_req.text) input('\nPress any key to close ...') exit() except IndexError as e: - output(2,'\n [13]ERROR','', 'Bad response from fansly API. Please make sure your configuration file is not malformed; most likely misspelled the creator name.') + output(2,'\n [15]ERROR','', 'Bad response from fansly API. Please make sure your configuration file is not malformed; most likely misspelled the creator name.') print('\n'+str(e)) print(raw_req.text) input('\nPress any key to close ...') @@ -139,7 +161,6 @@ def open_file(myfile): total_videos = acc_req['timelineStats']['videoCount'] output(1,' Info','','Targeted creator: "'+mycreator+'"') -output(1,' Info','','Using authorisation token: "'+mytoken+'"') output(1,' Info','','Using user-agent: "'+myuseragent[:28]+' [...] '+myuseragent[-35:]+'"') output(1,' Info','','Open download folder when finished, is set to: "'+openwhenfinished+'"') output(1,' Info','','Downloading files marked as preview, is set to: "'+previews+'"') @@ -151,19 +172,23 @@ def open_file(myfile): output(4,'\n lnfo','', f"Fansly scraper was downloaded {tot_downs} times, but only {round(requests.get('https://api.github.com/repos/avnsx/fansly', headers={'User-Agent':'Fansly Scraper'}).json()['stargazers_count']/tot_downs*100, 2)} % of You(!) have starred it\n{19*' '}Stars directly influence my willingness to continue maintaining the project\n{23*' '}Help the repository grow today, by leaving a star on it!") s(15) -recent_photobyte_hashes=[] -recent_videobyte_hashes=[] +recent_photobyte_hashes, recent_videobyte_hashes, basedir = [], [], mycreator+'_fansly' -basedir=mycreator+'_fansly' +def process_img(filePath): + recent_photobyte_hashes.append(str(imagehash.average_hash(Image.open(filePath)))) -def process_img(name): - recent_photobyte_hashes.append(str(imagehash.average_hash(Image.open(basedir+'/Pictures/'+name)))) - -def process_vid(name): - with open(basedir+'/Videos/'+name, 'rb') as f: +def process_vid(filePath): + with open(filePath, 'rb') as f: recent_videobyte_hashes.append(hashlib.md5(f.read()).hexdigest()) print('') +if remember == 'Auto': + output(1,' Info','', 'Automatically detecting whether download folder exists') + if os.path.isdir(basedir): + remember = 'True' + else: + remember = 'False' + if remember == 'True': if os.path.isdir(basedir): output(1,' Info','', f'"{basedir}" folder exists in local directory') @@ -175,46 +200,64 @@ def process_vid(name): if basedir: output(1,' Info','', f'Chose folder path {basedir}') else: - output(2,'\n [14]ERROR','', f'Could not register your chosen folder path, please start all over again. Closing in 30 seconds') + output(2,'\n [16]ERROR','', f'Could not register your chosen folder path, please start all over again. Closing in 30 seconds') s(30) exit() - # pictures - output(1,' Info','', f"Hashing {mycreator}'s recently downloaded pictures ...") list_of_futures=[] with concurrent.futures.ThreadPoolExecutor() as executor: - for el in os.listdir(basedir+'/Pictures'): - list_of_futures.append(executor.submit(process_img, el)) - concurrent.futures.wait(list_of_futures) - - # videos - output(1,' Info','', f"Hashing {mycreator}'s recently downloaded videos ...") - list_of_futures=[] - with concurrent.futures.ThreadPoolExecutor() as executor: - for el in os.listdir(basedir+'/Videos'): - list_of_futures.append(executor.submit(process_vid, el)) - concurrent.futures.wait(list_of_futures) + for x in '', '/Timeline', '/Messages', '/Previews', '/Timeline/Previews', '/Messages/Previews': + x_path = basedir + x + if os.path.isdir(x_path): + p_path = x_path + '/Pictures' + v_path = x_path + '/Videos' + if os.path.isdir(p_path): + output(1,' Info','', f"Hashing {mycreator}'s recently downloaded pictures from {p_path} ...") + for el in os.listdir(p_path): + list_of_futures.append(executor.submit(process_img, f'{p_path}/{el}')) + + if os.path.isdir(x_path + '/Videos'): + output(1,' Info','', f"Hashing {mycreator}'s recently downloaded videos from {v_path} ...") + for el in os.listdir(v_path): + list_of_futures.append(executor.submit(process_vid, f'{v_path}/{el}')) + + concurrent.futures.wait(list_of_futures) output(1,' Info','', f'Finished hashing! Will now compare each new download against {len(recent_photobyte_hashes)} photo & {len(recent_videobyte_hashes)} video hashes.') else: try: output(1,' Info','','Creating download directories ...') os.makedirs(basedir, exist_ok = True) - os.makedirs(basedir+'/Pictures', exist_ok = True) - os.makedirs(basedir+'/Videos', exist_ok = True) + + if seperate_messages == 'True': + os.makedirs(basedir+'/Messages', exist_ok = True) + os.makedirs(basedir+'/Messages/Pictures', exist_ok = True) + os.makedirs(basedir+'/Messages/Videos', exist_ok = True) + os.makedirs(basedir+'/Timeline', exist_ok = True) + os.makedirs(basedir+'/Timeline/Pictures', exist_ok = True) + os.makedirs(basedir+'/Timeline/Videos', exist_ok = True) + if previews == 'True' and seperate_previews == 'True': + os.makedirs(basedir+'/Messages/Previews', exist_ok = True) + os.makedirs(basedir+'/Messages/Previews/Pictures', exist_ok = True) + os.makedirs(basedir+'/Messages/Previews/Videos', exist_ok = True) + os.makedirs(basedir+'/Timeline/Previews', exist_ok = True) + os.makedirs(basedir+'/Timeline/Previews/Pictures', exist_ok = True) + os.makedirs(basedir+'/Timeline/Previews/Videos', exist_ok = True) + else: + os.makedirs(basedir+'/Pictures', exist_ok = True) + os.makedirs(basedir+'/Videos', exist_ok = True) + if seperate_previews == 'True': + os.makedirs(basedir+'/Previews', exist_ok = True) + os.makedirs(basedir+'/Previews/Pictures', exist_ok = True) + os.makedirs(basedir+'/Previews/Videos', exist_ok = True) except Exception: print('\n'+traceback.format_exc()) - output(2,'\n [15]ERROR','', 'Creating download directories ... Please copy & paste this on GitHub > Issues & provide a short explanation.') + output(2,'\n [17]ERROR','', 'Creating download directories ... Please copy & paste this on GitHub > Issues & provide a short explanation.') input('\nPress any key to close ...') exit() -pic_count=1 -vid_count=1 -duplicates=0 -recent=0 -photobyte_hashes=[] -videobyte_hashes=[] -def sort_download(filename,filebytes): +pic_count, vid_count, duplicates, recent, photobyte_hashes, videobyte_hashes = 1, 1, 0, 0, [], [] +def sort_download(filename,filebytes, directoryName): global pic_count, vid_count, duplicates, recent win_comp_name=str(re.sub(r'[\\/:*?"<>|]', '', repr(filename).replace("'",''))).replace('..','.') randints=''.join(choices(digits, k=3)) @@ -222,8 +265,12 @@ def sort_download(filename,filebytes): photohash=str(imagehash.average_hash(Image.open(io.BytesIO(filebytes)))) if photohash not in recent_photobyte_hashes: if photohash not in photobyte_hashes: + if naming == 'Datepost': + prefix = "" + else: + prefix = f"{pic_count}-{randints}_" if show == 'True':output(1,' Info','', f"Downloading Image '{win_comp_name}'") - with open(f"{basedir}/Pictures/{pic_count}-{randints}_{win_comp_name}", 'wb') as f:f.write(filebytes) + with open(f"{directoryName}/Pictures/{prefix}{win_comp_name}", 'wb') as f:f.write(filebytes) photobyte_hashes.append(photohash) pic_count+=1 else:duplicates+=1 @@ -232,18 +279,22 @@ def sort_download(filename,filebytes): videohash=hashlib.md5(filebytes).hexdigest() if videohash not in recent_videobyte_hashes: if videohash not in videobyte_hashes: + if naming == 'Datepost': + prefix = "" + else: + prefix = f"{vid_count}-{randints}_" if show == 'True':output(1,' Info','', f"Downloading Video '{win_comp_name}'") - with open(f"{basedir}/Videos/{vid_count}-{randints}_{win_comp_name}", 'wb') as f:f.write(filebytes) + with open(f"{directoryName}/Videos/{prefix}{win_comp_name}", 'wb') as f:f.write(filebytes) videobyte_hashes.append(videohash) vid_count+=1 else:duplicates+=1 else:recent+=1 else: - output(2,'\n [16]ERROR','', 'Unknown filetype: "'+str(filename[-7:])+'" please report this on GitHub > Issues & provide a short explanation; continuing without that file ...') + output(2,'\n [18]ERROR','', 'Unknown filetype: "'+str(filename[-7:])+'" please report this on GitHub > Issues & provide a short explanation; continuing without that file ...') # scrape messages group_id = None -groups = sess.get('https://apiv2.fansly.com/api/v1/group', headers=headers).json()['response']['groups'] +groups = sess.get('https://apiv3.fansly.com/api/v1/group', headers=headers).json()['response']['groups'] for x in range(len(groups)): if groups[x]['users'][0]['userId'] == creator_id: group_id = groups[x]['id'] @@ -253,32 +304,46 @@ def sort_download(filename,filebytes): output(1,' Info','','Started messages media download ...') msg_cursor = None while True: + if seperate_messages == 'True': + directory_name = f'{basedir}/messages' + else: + directory_name = f'{basedir}' + if seperate_previews == 'True': + preview_directory_name = f'{directory_name}/previews' + else: + preview_directory_name = directory_name if not msg_cursor: output(1,' Info','', f'Inspecting message: {group_id}') - resp = sess.get('https://apiv2.fansly.com/api/v1/message', headers=headers, params=(('groupId', group_id),('limit', '50'),)).json() + resp = sess.get('https://apiv3.fansly.com/api/v1/message', headers=headers, params=(('groupId', group_id),('limit', '50'),)).json() else: output(1,' Info','', f'Inspecting message: {msg_cursor}') - resp = sess.get('https://apiv2.fansly.com/api/v1/message', headers=headers, params=(('groupId', group_id),('before', msg_cursor),('limit', '50'),)).json() + resp = sess.get('https://apiv3.fansly.com/api/v1/message', headers=headers, params=(('groupId', group_id),('before', msg_cursor),('limit', '50'),)).json() try: for x in resp['response']['accountMedia']: + # set filename + if naming == 'Datepost': + ts = int(x['createdAt']) + file_datetime = datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H-%M-%S') + file_id = x['id'] + file_name = f"{file_datetime} {file_id}" + else: + file_name = f"{mycreator}" # message media previews if previews == 'True': try: - if x['access'] != False: - sort_download(f"{mycreator}.{x['media']['mimetype'].split('/')[1]}", sess.get(x['preview']['locations'][0]['location'], headers=headers).content) - if x['access'] == False: - sort_download(f"{mycreator}.png", sess.get(x['preview']['locations'][0]['location'], headers=headers).content) + sort_download(f"{file_name} preview.{x['preview']['mimetype'].split('/')[1]}", sess.get(x['preview']['locations'][0]['location'], headers=headers).content, preview_directory_name) except:pass # unlocked meda in messages try: - locurl=x['media']['locations'][0]['location'] - sort_download(f"{mycreator}.{x['media']['mimetype'].split('/')[1]}", sess.get(locurl, headers=headers).content) + if(x['access'] == True): + locurl=x['media']['locations'][0]['location'] + sort_download(f"{file_name}.{x['media']['mimetype'].split('/')[1]}", sess.get(locurl, headers=headers).content, directory_name) # unlocked messages without corresponding location url except IndexError: for f in range(0,len(x['media']['variants'])): try: locurl=x['media']['variants'][f]['locations'][0]['location'] - sort_download(f"{mycreator}.{x['media']['mimetype'].split('/')[1]}", sess.get(locurl, headers=headers).content) + sort_download(f"{file_name}.{x['media']['mimetype'].split('/')[1]}", sess.get(locurl, headers=headers).content, directory_name) break except:pass # silently passing locked media in messages pass @@ -289,7 +354,7 @@ def sort_download(filename,filebytes): except IndexError:break # break if end is reached except Exception: print('\n'+traceback.format_exc()) - output(2,'\n [17]ERROR','', 'Please copy & paste this on GitHub > Issues & provide a short explanation.') + output(2,'\n [19]ERROR','', 'Please copy & paste this on GitHub > Issues & provide a short explanation.') input('\nPress any key to close ...') exit() except KeyError: @@ -300,29 +365,43 @@ def sort_download(filename,filebytes): output(1,' Info','','Started profile media download; this could take a while dependant on the content size ...') cursor = 0 while True: + if seperate_messages == 'True': + directory_name = f'{basedir}/timeline' + else: + directory_name = f'{basedir}' + if seperate_previews == 'True': + preview_directory_name = f'{directory_name}/previews' + else: + preview_directory_name = directory_name if cursor == 0:output(1,' Info','', f'Inspecting most recent page') else:output(1,' Info','', f'Inspecting page: {cursor}') - response = sess.get('https://apiv2.fansly.com/api/v1/timeline/'+creator_id+'?before='+str(cursor)+'&after=0', headers=headers) + response = sess.get('https://apiv3.fansly.com/api/v1/timeline/'+creator_id+'?before='+str(cursor)+'&after=0', headers=headers) try: for x in response.json()['response']['accountMedia']: + # set filename + if naming == 'Datepost': + ts = int(x['createdAt']) + file_datetime = datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H-%M-%S') + file_id = x['id'] + file_name = f"{file_datetime} {file_id}" + else: + file_name = f"{mycreator}" # previews if previews == 'True': try: - if x['access'] != False: - sort_download(f"{mycreator}.{x['media']['mimetype'].split('/')[1]}", sess.get(x['preview']['locations'][0]['location'], headers=headers).content) - if x['access'] == False: - sort_download(f"{mycreator}.png", sess.get(x['preview']['locations'][0]['location'], headers=headers).content) + sort_download(f"{file_name} preview.{x['preview']['mimetype'].split('/')[1]}", sess.get(x['preview']['locations'][0]['location'], headers=headers).content, preview_directory_name) except:pass # unlocked media try: - locurl=x['media']['locations'][0]['location'] - sort_download(f"{mycreator}.{x['media']['mimetype'].split('/')[1]}", sess.get(locurl, headers=headers).content) + if(x['access'] == True): + locurl=x['media']['locations'][0]['location'] + sort_download(f"{file_name}.{x['media']['mimetype'].split('/')[1]}", sess.get(locurl, headers=headers).content, directory_name) # unlocked media without corresponding location url except IndexError: for f in range(0,len(x['media']['variants'])): try: locurl=x['media']['variants'][f]['locations'][0]['location'] - sort_download(f"{mycreator}.{x['media']['mimetype'].split('/')[1]}", sess.get(locurl, headers=headers).content) + sort_download(f"{file_name}.{x['media']['mimetype'].split('/')[1]}", sess.get(locurl, headers=headers).content, directory_name) break except:pass # silently passing locked media pass @@ -333,11 +412,11 @@ def sort_download(filename,filebytes): except IndexError:break # break if end is reached except Exception: print('\n'+traceback.format_exc()) - output(2,'\n [18]ERROR','', 'Please copy & paste this on GitHub > Issues & provide a short explanation.') + output(2,'\n [20]ERROR','', 'Please copy & paste this on GitHub > Issues & provide a short explanation.') input('\nPress any key to close ...') exit() except KeyError: - output(2,'\n [19]ERROR','', "Couldn't find any scrapeable media at all!\n This most likely happend because you're not following the creator, your authorisation token is wrong\n or the creator is not providing unlocked content.") + output(2,'\n [21]ERROR','', "Couldn't find any scrapeable media at all!\n This most likely happend because you're not following the creator, your authorisation token is wrong\n or the creator is not providing unlocked content.") input('\nPress any key to close ...') exit() if remember == 'True' and recent > int(total_photos+total_videos) * 0.2: diff --git a/requirements.txt b/requirements.txt index 39fbd37..4d4e470 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ pillow>=8.4.0 python-dateutil>=2.8.2 psutil>=5.9.0 keyboard>=0.13.5 -selenium-wire==4.6.0 -undetected_chromedriver==3.0.6 +undetected_chromedriver>=3.1.5 pycryptodome>=3.14.0 pywin32>=303