update

README.md

@@ -19,31 +19,31 @@ H是一款强大的资产收集管理平台，主要用于src信息收集和红
 ### 开发日志
 闲暇时间龟速开发，想起了就记录下进度
 
-2021.9.10 -- 搭建好了前端，罗列好了目录，使用框架(https://github.com/app-generator/flask-datta-able) 
+2021.1.17 -- 修复暂停(停止)进度条卡住的bug，shuffledns子域名爆破采用多线程提高效率，修复截图和指纹识别选项卡错误，修复扫描结束进度条卡在漏洞扫描处bug，修复了项目详细中域名资产统计bug，优化项目暂停时发起celery任务终止请求
 
-2021.9.13 -- 数据库交互确认，数据渲染逻辑确认，完成资产管理页面，资产添加页面(50%)，扫描模式添加页面(25%)
+2021.11.22 -- 上传到github，目前在每日新poc扫描模块和是否是新资产识别存在一定的bug，但并不影响整个项目运行
 
-2021.9.23 -- 完成资产添加，资产管理，扫描模式，扫描周期，系统配置，资产详细(25%)
+2021.11.15 -- 修复一堆bug
 
-2021.9.27 -- 完成资产的增删改，完成域名爆破，端口扫描，web扫描，目录扫描的扫描逻辑
+2021.11.8 -- 完成整体代码，打包成功运行，后续可能有bug再做调整
 
-2021.9.29 -- 完成所有扫描逻辑, 完成扫描结果页面, 待完成事项:主页,项目详细页面,站点详细页面,邮件推送功能,快捷插件
+2021.10.29 -- 完成备案查询，一键获取域名和子公司域名
 
-2021.10.11 -- 完成简单主页，完成项目详细页面(全网段信息统计待定),站点详细页面，待完成：邮件推送功能,快捷插件
+2021.10.27 -- 完成host碰撞插件，完成apk域名提取插件，完成邮件推送，完成定时任务功能
 
 2021.10.18 -- 完成docker镜像的构建，具体细节有待调整，计划实现插件功能
 
-2021.10.27 -- 完成host碰撞插件，完成apk域名提取插件，完成邮件推送，完成定时任务功能
-
-2021.10.29 -- 完成备案查询，一键获取域名和子公司域名
+2021.10.11 -- 完成简单主页，完成项目详细页面(全网段信息统计待定),站点详细页面，待完成：邮件推送功能,快捷插件
 
-2021.11.8 -- 完成整体代码，打包成功运行，后续可能有bug再做调整
+2021.9.29 -- 完成所有扫描逻辑, 完成扫描结果页面, 待完成事项:主页,项目详细页面,站点详细页面,邮件推送功能,快捷插件
 
-2021.11.15 -- 修复一堆bug
+2021.9.27 -- 完成资产的增删改，完成域名爆破，端口扫描，web扫描，目录扫描的扫描逻辑
 
-2021.11.22 -- 上传到github，目前在每日新poc扫描模块和是否是新资产识别存在一定的bug，但并不影响整个项目运行
+2021.9.23 -- 完成资产添加，资产管理，扫描模式，扫描周期，系统配置，资产详细(25%)
 
+2021.9.13 -- 数据库交互确认，数据渲染逻辑确认，完成资产管理页面，资产添加页面(50%)，扫描模式添加页面(25%)
 
+2021.9.10 -- 搭建好了前端，罗列好了目录，使用框架(https://github.com/app-generator/flask-datta-able) 
 
 ### 系统搭建
 

app/base/routes.py

@@ -99,7 +99,6 @@ def changepassword():
                 return render_template( 'accounts/changepassword.html', msg='新密码两次输入不一样', form=change_account_form)
 
             user_result = queryToDict(user)
-            print(user_result)
             user_result['password'] = hash_pass(request.form['newpassword1'])
 
             db.session.query(User).filter(User.username == str(current_user)).update(user_result)

app/home/dirb/view.py

@@ -17,7 +17,6 @@ def dirb(DynamicModel = Dirb):
 
     #查询
     search = request.args.get('search')
-    print(search)
     if search != 'None' and search and '=' in search:
         target = ""
         dir_url = ""

app/home/http/view.py

@@ -59,7 +59,6 @@ def http(DynamicModel = Http):
             if('new' in i):
                 new = 0 if i.split("=")[1] == 'true' else 2
 
-        print(new)
         search = search.replace("&&", "%26%26")
         if(is_admin()):
             query = db.session.query(

app/home/port/view.py

@@ -17,7 +17,6 @@ def port(DynamicModel = Port):
 
     #查询
     search = request.args.get('search')
-    print(search)
     if search != 'None' and search and '=' in search:
         target = ""
         port_domain = ""

app/home/scanconfig/view.py

@@ -63,7 +63,6 @@ def scancron(DynamicModel = Scancron, DynamicFrom = ScancronFrom):
     if request.method == 'POST':
         tmpform = request.form.to_dict()
         scancron = utils.form_to_model(tmpform, DynamicModel())
-        print(scancron)
         if(id):
             if('delete' in tmpform):
                 db.session.query(DynamicModel).filter(DynamicModel.id == id).delete()

app/home/subdomain/view.py

@@ -17,7 +17,6 @@ def subdomain(DynamicModel = Subdomain):
 
     #查询
     search = request.args.get('search')
-    print(search)
     if search != 'None' and search and '=' in search:
         target = ""
         start_time = "2021-01-01 00:00:00"
@@ -42,7 +41,6 @@ def subdomain(DynamicModel = Subdomain):
                 subdomain_ip = i.split("=")[1]
             if('new' in i):
                 new = 0 if i.split("=")[1] == 'true' else 1
-            print(i)
 
         search = search.replace("&&", "%26%26")
         if(is_admin()):

app/home/target/models.py

@@ -23,3 +23,9 @@ class Blacklist(db.Model):
     black_name = Column(String(128), unique=True)   #黑名单内容，以domain,ip,title三类标志
     black_time = Column(String(128))   #修改时间
     black_target = Column(Integer) #隶属于的目标
+
+class Celerytask(db.Model):
+    __tablename__ = 'Celerytask'
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    celery_target = Column(Integer)   #celery对应的项目id
+    celery_id = Column(String(128))   #是否是允许运行的

app/home/target/view.py

@@ -121,7 +121,6 @@ def target(DynamicModel = Target):
 
     #查询
     search = request.args.get('search')
-    print(search)
     if search != 'None' and search and '=' in search:
         target = ""
         start_time = "2021-01-01 00:00:00"
@@ -137,7 +136,6 @@ def target(DynamicModel = Target):
                 end_time = i.split("=")[1]
             if('user' in i):
                 user = i.split("=")[1]
-            print(i)
 
         search = search.replace("&&", "%26%26")
         if(is_admin()):
@@ -230,7 +228,6 @@ def targetadd(DynamicModel = Target, form = TargetForm):
 
     #处理发送添加请求
     if request.method == 'POST':
-        print()
         tmpform = request.form.to_dict()
         target = form_to_model(tmpform, DynamicModel())
 
@@ -331,8 +328,8 @@ def targetinfo(DynamicModel = Target, DynamicFrom = TargetForm):
 
     vuln_count = Vuln.query.filter(Vuln.vuln_target == id).count()
     web_count = Http.query.filter(Http.http_target == id).count()
-    old_domain = Subdomain.query.filter(Subdomain.subdomain_new == 1).count()
-    new_domain = Subdomain.query.filter(Subdomain.subdomain_new == 0).count()
+    old_domain = Subdomain.query.filter(Subdomain.subdomain_new == 1, Subdomain.subdomain_target == id).count()
+    new_domain = Subdomain.query.filter(Subdomain.subdomain_new == 0, Subdomain.subdomain_target == id).count()
     status_200 = Http.query.filter(Http.http_target == id, Http.http_status == '200').count()
     status_30x = Http.query.filter(Http.http_target == id, Http.http_status.like('%30%')).count()
     status_50x = Http.query.filter(Http.http_target == id, Http.http_status.like('%50%')).count()
@@ -408,7 +405,6 @@ def targetedit(DynamicModel = Target, DynamicFrom = TargetForm):
 
     #处理发送添加请求
     if request.method == 'POST':
-        print()
         tmpform = request.form.to_dict()
         target = form_to_model(tmpform, DynamicModel())
 

app/home/templates/target.html

@@ -108,7 +108,7 @@ <h5>资产列表</h5>
                                                             <div class="progress-bar progress-c-theme" role="progressbar" style="width: 65%;" aria-valuenow="70" aria-valuemin="0" aria-valuemax="100"></div>
                                                         {% elif row.target_status == 6 %}
                                                             <div class="progress-bar progress-c-theme" role="progressbar" style="width: 79%;" aria-valuenow="70" aria-valuemin="0" aria-valuemax="100"></div>
-                                                        {% else %}
+                                                        {% elif row.target_status == 7 %}
                                                             <div class="progress-bar progress-c-theme" role="progressbar" style="width: 100%;" aria-valuenow="70" aria-valuemin="0" aria-valuemax="100"></div>
                                                         {% endif %}
                                                     </div>

app/home/vuln/view.py

@@ -17,7 +17,6 @@ def vuln(DynamicModel = Vuln):
 
     #查询
     search = request.args.get('search')
-    print(search)
     if search != 'None' and search and '=' in search:
         target = ""
         vuln_url = ""

app/scan/lib/Scandir.py

@@ -47,6 +47,10 @@ def scan_dir(scanmethod_query, target_id, current_user):
         for j in threads:
             j.join()
 
+        sql = "DELETE FROM Celerytask WHERE celery_target= %s"
+        cursor.execute(sql,(target_id,))
+        conn.commit()
+
     if(scanmethod_query[11] == True):
         #12是字典
         wordlist = scanmethod_query[12]
@@ -59,6 +63,10 @@ def scan_dir(scanmethod_query, target_id, current_user):
         for j in threads:
             j.join()
 
+        sql = "DELETE FROM Celerytask WHERE celery_target= %s"
+        cursor.execute(sql,(target_id,))
+        conn.commit()
+
     #关闭数据库句柄
     cursor.close()
     conn.close()
@@ -86,6 +94,11 @@ def run(self):
             target = queue.get()
             scan_target = target[1] + ':https://' + target[2]
             dir_scan = task.send_task('jsfinder.run', args=(scan_target,), queue='jsfinder')
+            sql = "INSERT INTO Celerytask(celery_target, celery_id) VALUES(%s,%s)"
+            lock.acquire()
+            cursor.execute(sql,(target_id, dir_scan.id,))
+            conn.commit()
+            lock.release()
             while True:
                 if dir_scan.successful():
                     try:
@@ -97,6 +110,15 @@ def run(self):
                         print(e)
                         break
 
+                sql = "SELECT * FROM Celerytask where celery_target = %s"
+                cursor.execute(sql,(target_id,))
+                celery_status = cursor.fetchone()[2]
+
+                if celery_status == False:
+                    task.control.revoke(dir_scan.id, terminate=True)
+                    break
+                time.sleep(2)
+
 
 class tool_fileleak(Thread):
     def __init__(self, fileleak_queue, task, wordlist,target_id, conn, cursor, current_user):
@@ -122,6 +144,11 @@ def run(self):
             target = queue.get()
             scan_target = target[1] + ':https://' + target[2]
             dir_scan = task.send_task('fileleak.run', args=(scan_target,wordlist,), queue='fileleak')
+            sql = "INSERT INTO Celerytask(celery_target, celery_id) VALUES(%s,%s)"
+            lock.acquire()
+            cursor.execute(sql,(target_id, dir_scan.id,))
+            conn.commit()
+            lock.release()
             while True:
                 if dir_scan.successful():
                     try:
@@ -133,6 +160,15 @@ def run(self):
                         print(e)
                         break
 
+                sql = "SELECT * FROM Celerytask where celery_target = %s"
+                cursor.execute(sql,(target_id,))
+                celery_status = cursor.fetchone()[2]
+
+                if celery_status == False:
+                    task.control.revoke(dir_scan.id, terminate=True)
+                    break
+                time.sleep(2)
+
 #保存
 def save_result(target, target_id, result, cursor, conn, current_user): 
     tool = result['tool']

app/scan/lib/Scanhttp.py

@@ -40,10 +40,10 @@ def scan_http(scanmethod_query, target_id, current_user):
     if(scanmethod_query[7] == True):
         tool_httpx(task, subdomain_list, target_id, conn, cursor, current_user)
     #截图     
-    if(scanmethod_query[8] == True):
+    if(scanmethod_query[9] == True):
         tool_screenshot(task, target_id, conn, cursor)
     #获取指纹
-    if(scanmethod_query[9] == True):
+    if(scanmethod_query[8] == True):
         tool_ehole(task, target_id, conn, cursor)
 
     cursor.close()
@@ -58,16 +58,20 @@ def tool_httpx(task, subdomain_list, target_id, conn, cursor, current_user):
 
     while(len(subdomain_list)):
         httpx_scan = task.send_task('httpx.run', args=(sub_list,), queue='httpx')
+        sql = "INSERT INTO Celerytask(celery_target, celery_id) VALUES(%s,%s)"
+        cursor.execute(sql,(target_id, httpx_scan.id,))
+        conn.commit()
         while True:
             if httpx_scan.successful():
                 try:
                     save_result(target_id, httpx_scan.result['result'], cursor, conn, current_user)
                 except Exception as e:
                     print(e)
                 finally:
+                    sql = "DELETE FROM Celerytask WHERE celery_id= %s"
+                    cursor.execute(sql,(httpx_scan.id,))
+                    conn.commit()
                     break
-
-
         subdomain_list = subdomain_list[100:]
         sub_list = subdomain_list[0:100] if len(subdomain_list) > 100 else subdomain_list
 
@@ -85,14 +89,21 @@ def tool_screenshot(task, target_id, conn, cursor):
             http_list.append(http_info[1] + ":https://" + http_info[2])
 
         screenshot_scan = task.send_task('screenshot.run', args=(http_list,), queue='screenshot')
+        sql = "INSERT INTO Celerytask(celery_target, celery_id) VALUES(%s,%s)"
+        cursor.execute(sql,(target_id, screenshot_scan.id,))
+        conn.commit()
         while True:
             if screenshot_scan.successful():
                 try:
                     save_result_screenshot(screenshot_scan.result['result'], cursor, conn)
                 except Exception as e:
                     print(e)
                 finally:
+                    sql = "DELETE FROM Celerytask WHERE celery_id= %s"
+                    cursor.execute(sql,(screenshot_scan.id,))
+                    conn.commit()
                     break
+
         screen_count = cursor.execute(sql,(target_id,'No','302','301'))
         http_query_all = cursor.fetchall()
     return
@@ -111,14 +122,21 @@ def tool_ehole(task, target_id, conn, cursor):
             conn.commit()
 
         finger_scan = task.send_task('ehole.run', args=(http_list,), queue='ehole')
+        sql = "INSERT INTO Celerytask(celery_target, celery_id) VALUES(%s,%s)"
+        cursor.execute(sql,(target_id, finger_scan.id,))
+        conn.commit()
         while True:
             if finger_scan.successful():
                 try:
                     save_result_finger(finger_scan.result['result'], cursor, conn)
                 except Exception as e:
-                    print(e)
+                    print(e) 
                 finally:
+                    sql = "DELETE FROM Celerytask WHERE celery_id= %s"
+                    cursor.execute(sql,(finger_scan.id,))
+                    conn.commit()
                     break
+
         finger_count = cursor.execute(sql,(target_id,'No',))
         http_query_all = cursor.fetchall()
 

app/scan/lib/Scanport.py

@@ -54,6 +54,10 @@ def scan_port(scanmethod_query, target_id, current_user):
     for j in threads:
         j.join()
 
+    sql = "DELETE FROM Celerytask WHERE celery_target= %s"
+    cursor.execute(sql,(target_id,))
+    conn.commit()
+
     cursor.close()
     conn.close()
 
@@ -87,6 +91,11 @@ def run(self):
                 #发送celery
                 #naabu + nmap
                 naabu_scan = task.send_task('naabu.run', args=(target, config), queue='naabu')
+                sql = "INSERT INTO Celerytask(celery_target, celery_id) VALUES(%s,%s)"
+                lock.acquire()
+                cursor.execute(sql,(target_id, naabu_scan.id,))
+                conn.commit()
+                lock.release()
                 while True:
                     if naabu_scan.successful():
                         try: