Merge pull request #22 from KAnanev/dev

Dev

page_analyzer/services/check_url.py

@@ -3,7 +3,7 @@
 import requests
 
 from page_analyzer.services.url import URLService
-
+from bs4 import BeautifulSoup
 from page_analyzer.models import URLChecks
 from page_analyzer.services.db import PostgresDB
 
@@ -36,6 +36,30 @@ def check_url(self, url_id):
  url_service = URLService(self.db)
  url = url_service.get_json_by_id(url_id).name
  if url:
- request = requests.get(url)
- if request:
- return URLChecks(url_id=url_id, status_code=request.status_code)
+ response = self.response_check(url)
+ if response:
+ return URLChecks(
+ url_id=url_id,
+ status_code=response['status_code'],
+ h1=response['h1'],
+ title=response['title'],
+ description=response['description'],
+ )
+
+ @staticmethod
+ def response_check(url):
+ response = requests.get(url)
+ soup = BeautifulSoup(response.text, 'html.parser')
+ h1_tag = soup.find('h1').text if soup.find('h1') else None
+ title = soup.find('title').text if soup.find('title') else None
+ description_tag = soup.find('meta', attrs={'name': 'description'})
+
+ description = description_tag[
+ 'content'] if description_tag else None
+
+ return {
+ 'status_code': response.status_code,
+ 'h1': h1_tag,
+ 'title': title,
+ 'description': description,
+ }

page_analyzer/services/url.py

@@ -80,7 +80,6 @@ def get_all_urls(self) -> List[URLModel] | None:
  items = self.db.execute_query(GET_ITEMS, many=True)
  if items:
  sorted_items = sorted(items, key=lambda item: -item['result']['id'])
- print(sorted_items)
  items = [URLSModel(**item['result']) for item in sorted_items]
  return items
 

page_analyzer/templates/urls.html

@@ -25,7 +25,7 @@
  </td>
  {% if item.url_checks %}
  {% for url_checks in item.url_checks %}
- <td>{{ url_checks.created_at }}</td>
+ <td>{{ url_checks.created_at.strftime('%Y-%m-%d') }}</td>
  <td>{{ url_checks.status_code }}</td>
  {% endfor %}
  {% else %}

pyproject.toml

@@ -17,6 +17,7 @@ psycopg-c = "^3.1.9"
 psycopg = "^3.1.9"
 pydantic = "^2.0"
 requests = "^2.31.0"
+beautifulsoup4 = "^4.12.2"
 
 
 [tool.poetry.group.dev.dependencies]
@@ -32,7 +33,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.flake8]
 accept-encodings = "utf-8"
-max-complexity = 6
+max-complexity = 5
 statistics = "False"
 max-line-length = 80
 enable-extensions = "G"