add upload progress

helpers.py

@@ -1,10 +1,15 @@
 import shutil
 import requests
 from utils import FINISHED_PROGRESS_STR, UN_FINISHED_PROGRESS_STR
+from pyrogram.types import InlineKeyboardMarkup, InlineKeyboardButton
 import os
 from selenium import webdriver
 from selenium.webdriver.firefox.options import Options as FirefoxOptions
 from selenium.webdriver.chrome.options import Options
+import math
+import os
+import time
+
 
 async def progress_bar(current, total):
  percentage = current / total
@@ -15,6 +20,75 @@ async def progress_bar(current, total):
  return progress, formatted_percentage
 
 
+async def progress_for_pyrogram(current, total, ud_type, message, start):
+ reply_markup = InlineKeyboardMarkup(
+ [[InlineKeyboardButton("🚫Cancel", callback_data="cdstoptrasmission")]]
+ )
+ now = time.time()
+ diff = now - start
+ if round(diff % 10.00) == 0 or current == total:
+ # if round(current / total * 100, 0) % 5 == 0:
+ percentage = current * 100 / total
+ speed = current / diff
+ elapsed_time = round(diff) * 1000
+ time_to_completion = round((total - current) / speed) * 1000
+ estimated_total_time = elapsed_time + time_to_completion
+
+ elapsed_time = TimeFormatter(milliseconds=elapsed_time)
+ estimated_total_time = TimeFormatter(milliseconds=estimated_total_time)
+
+ progress = "[{0}{1}] \n <b>📊Percentage:</b> {2}%\n".format(
+ "".join(["■" for i in range(math.floor(percentage / 5))]),
+ "".join(["□" for i in range(20 - math.floor(percentage / 5))]),
+ round(percentage, 2),
+ )
+
+ tmp = (
+ progress
+ + "<b>✅Completed:</b>{0} \n<b>📁Total Size:</b> {1}\n<b>🚀Speed:</b> {2}/s\n<b>⌚️ETA:</b> {3}\n @BughunterBots".format(
+ humanbytes(current),
+ humanbytes(total),
+ humanbytes(speed),
+ estimated_total_time if estimated_total_time != "" else "0 s",
+ )
+ )
+ try:
+ await message.edit(
+ text="{}\n {}".format(ud_type, tmp), reply_markup=reply_markup
+ )
+ except:
+ pass
+
+
+def humanbytes(size):
+ # https://stackoverflow.com/a/49361727/4723940
+ # 2**10 = 1024
+ if not size:
+ return ""
+ power = 2**10
+ n = 0
+ Dic_powerN = {0: " ", 1: "Ki", 2: "Mi", 3: "Gi", 4: "Ti"}
+ while size > power:
+ size /= power
+ n += 1
+ return str(round(size, 2)) + " " + Dic_powerN[n] + "B"
+
+
+def TimeFormatter(milliseconds: int) -> str:
+ seconds, milliseconds = divmod(int(milliseconds), 1000)
+ minutes, seconds = divmod(seconds, 60)
+ hours, minutes = divmod(minutes, 60)
+ days, hours = divmod(hours, 24)
+ tmp = (
+ ((str(days) + "d, ") if days else "")
+ + ((str(hours) + "h, ") if hours else "")
+ + ((str(minutes) + "m, ") if minutes else "")
+ + ((str(seconds) + "s, ") if seconds else "")
+ + ((str(milliseconds) + "ms, ") if milliseconds else "")
+ )
+ return tmp[:-2]
+
+
 async def download_image(base_url, image_url, idx):
  try:
  r = requests.get(image_url, stream=True)
@@ -94,7 +168,9 @@ async def init_headless_browser(url):
  driver_chrome.get(url)
  return driver_chrome
  except Exception as e_chrome:
- print(f"An error occurred while initializing the Chrome headless browser: {e_chrome}")
+ print(
+ f"An error occurred while initializing the Chrome headless browser: {e_chrome}"
+ )
 
  try:
  options_firefox = FirefoxOptions()
@@ -103,5 +179,7 @@ async def init_headless_browser(url):
  driver_firefox.get(url)
  return driver_firefox
  except Exception as e_firefox:
- print(f"An error occurred while initializing the Firefox headless browser: {e_firefox}")
+ print(
+ f"An error occurred while initializing the Firefox headless browser: {e_firefox}"
+ )
  return None

main.py

@@ -53,7 +53,7 @@ async def start(_, message: Message):
 
 
 @app.on_callback_query()
-async def cb_data(bot, update):
+async def cb_data(bot:Client, update):
  if update.data == "cbrdata":
  await raw_data_scraping(update)
  elif update.data == "cbhtmldata":
@@ -80,6 +80,8 @@ async def cb_data(bot, update):
  await capture_screenshot(update)
  elif update.data == "cbscreenrecord":
  await record_screen(update)
+ elif update.data == "cdstoptrasmission":
+ bot.stop_transmission()
  elif update.data == 'cbcrawl':
  if CRAWL_LOG_CHANNEL:
  await crawl_web(bot,update)

scraper.py

@@ -13,8 +13,10 @@
  download_pdf,
  init_headless_browser,
  progress_bar,
+ progress_for_pyrogram
 )
 import imageio
+from tqdm import tqdm
 
 
 async def scrape(url):
@@ -144,6 +146,10 @@ async def all_paragraph_scraping(query):
  )
  return e
 
+async def progress_callback(current, total):
+ print("🎁 ~ scraper.py:148 -> current: ", current)
+ print("🛵 ~ scraper.py:148 -> total: ", total)
+
 
 async def all_images_scraping(query):
  try:
@@ -181,10 +187,12 @@ async def all_images_scraping(query):
  await status.edit("Uploading ....")
  zip_filename = f"{message.chat.id}-images.zip"
  shutil.make_archive(folder_name, "zip", folder_name)
-
+ c_time = time.time()
  await message.reply_document(
  open(zip_filename, "rb"),
  caption="Here are the images! \n @BUghunterBots",
+ progress=progress_for_pyrogram,
+ progress_args=('Uploading',status,c_time) 
  )
  await status.delete()
  await text.delete()
@@ -239,10 +247,12 @@ async def all_audio_scraping(query):
  await status.edit("Uploading ....")
  zip_filename = f"{message.chat.id}-audios.zip"
  shutil.make_archive(folder_name, "zip", folder_name)
-
+ c_time = time.time()
  await message.reply_document(
  open(zip_filename, "rb"),
- caption="Here are the images! \n @BUghunterBots",
+ caption="Here are the images! \n @BughunterBots",
+ progress=progress_for_pyrogram,
+ progress_args=('Uploading',status,c_time) 
  )
  await status.delete()
  await txt.delete()
@@ -307,16 +317,18 @@ async def all_video_scraping(query):
  with open(os.path.join(folder_name, local_filename), "wb") as file:
  file.write(video_data)
 
- # await status.edit(f"Downloaded {idx + 1}/{len(video_links)}")
  time.sleep(0.3)
 
  await status.edit("Uploading ....")
  zip_filename = f"{folder_name}.zip"
  shutil.make_archive(folder_name, "zip", folder_name)
 
+ c_time = time.time()
  await message.reply_document(
  open(zip_filename, "rb"),
  caption="Here are the videos! \n @BughunterBots",
+ progress=progress_for_pyrogram,
+ progress_args=('Uploading',status,c_time) 
  )
  # await message.reply_video(local_filename)
  await status.delete()
@@ -387,8 +399,11 @@ async def all_pdf_scraping(query):
  zip_filename = f"{folder_name}.zip"
  shutil.make_archive(folder_name, "zip", folder_name)
 
+ c_time = time.time()
  await message.reply_document(
- open(zip_filename, "rb"), caption="Here are the Pdfs! \n @BughunterBots"
+ open(zip_filename, "rb"), caption="Here are the Pdfs! \n @BughunterBots",
+ progress=progress_for_pyrogram,
+ progress_args=('Uploading',status,c_time) 
  )
  await status.delete()
  await txt.delete()