-
-
Notifications
You must be signed in to change notification settings - Fork 1.6k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
WIP: Storing records on remote s3 storage #6586
base: dev
Are you sure you want to change the base?
Changes from all commits
5f4a107
f60af0f
07cdec0
361e860
d257fd2
1dc78c8
502baec
33c8126
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,8 +4,15 @@ | |
from pathlib import Path | ||
import shutil | ||
import threading | ||
|
||
import boto3 | ||
from botocore import UNSIGNED | ||
from botocore.config import Config | ||
from botocore.session import Session as boto_session | ||
from botocore.exceptions import BotoCoreError, ClientError | ||
from peewee import fn | ||
import os | ||
import tempfile | ||
|
||
|
||
from frigate.config import FrigateConfig | ||
from frigate.const import RECORD_DIR | ||
|
@@ -17,6 +24,98 @@ | |
) | ||
|
||
|
||
class StorageS3: | ||
def __init__(self, config: FrigateConfig) -> None: | ||
self.config = config | ||
if self.config.storage.s3.enabled or self.config.storage.s3.archive: | ||
if self.config.storage.s3.endpoint_url.startswith("http:https://"): | ||
try: | ||
session = boto_session() | ||
session.set_config_variable( | ||
"s3", | ||
{ | ||
"use_ssl": False, | ||
"verify": False, | ||
}, | ||
) | ||
self.s3_client = session.create_client( | ||
"s3", | ||
aws_access_key_id=self.config.storage.s3.access_key_id, | ||
aws_secret_access_key=self.config.storage.s3.secret_access_key, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You are forcing the use of keys. If See: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html |
||
endpoint_url=self.config.storage.s3.endpoint_url, | ||
config=Config(), | ||
) | ||
except (BotoCoreError, ClientError) as error: | ||
logger.error(f"Failed to create S3 client: {error}") | ||
return None | ||
else: | ||
try: | ||
self.s3_client = boto3.client( | ||
"s3", | ||
aws_access_key_id=self.config.storage.s3.access_key_id, | ||
aws_secret_access_key=self.config.storage.s3.secret_access_key, | ||
endpoint_url=self.config.storage.s3.endpoint_url, | ||
) | ||
except (BotoCoreError, ClientError) as error: | ||
logger.error(f"Failed to create S3 client: {error}") | ||
return None | ||
|
||
self.s3_bucket = self.config.storage.s3.bucket_name | ||
self.s3_path = self.config.storage.s3.path | ||
|
||
def upload_file_to_s3(self, file_path) -> str: | ||
try: | ||
s3_filename = self.s3_path + "/" + os.path.relpath(file_path, RECORD_DIR) | ||
self.s3_client.upload_file(file_path, self.s3_bucket, s3_filename) | ||
logger.debug(f"Uploading {file_path} to S3 {s3_filename}") | ||
except Exception as e: | ||
logger.error( | ||
f"Error occurred while uploading {file_path} to S3 {s3_filename}: {e}" | ||
) | ||
return "" | ||
return s3_filename | ||
|
||
def download_file_from_s3(self, s3_file_name) -> str: | ||
if self.config.storage.s3.enabled or self.config.storage.s3.archive: | ||
# Create a temporary directory | ||
temp_dir = tempfile.gettempdir() | ||
|
||
# Create a temporary file name with the same name as the original S3 file | ||
local_file_path = os.path.join(temp_dir, os.path.basename(s3_file_name)) | ||
|
||
try: | ||
# Download the file from S3 | ||
self.s3_client.download_file( | ||
self.s3_bucket, s3_file_name, local_file_path | ||
) | ||
logger.debug(f"Downloaded {s3_file_name} to {local_file_path}") | ||
return local_file_path | ||
except Exception as e: | ||
logger.error( | ||
f"Error occurred while downloading {s3_file_name} from S3: {e}" | ||
) | ||
return None | ||
else: | ||
return False | ||
|
||
def get_bucket_stats(self): | ||
try: | ||
total_size = 0 | ||
total_files = 0 | ||
for obj in self.s3_client.list_objects(Bucket=self.s3_bucket).get( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You should use Something along those liines: import boto3
def calculate_s3_bucket_size_and_file_count(bucket_name):
s3 = boto3.client('s3')
total_size = 0
total_files = 0
paginator = s3.get_paginator('list_objects_v2')
for page in paginator.paginate(Bucket=bucket_name):
for obj in page['Contents']:
total_size += obj['Size']
total_files += 1
return total_size, total_files
bucket_name = 'my-bucket' # Replace with your bucket name
total_size, total_files = calculate_s3_bucket_size_and_file_count(bucket_name)
print(f'Total size: {total_size / 1024**3} GB') # Convert bytes to GB
print(f'Total number of files: {total_files}') |
||
"Contents", [] | ||
): | ||
total_size += obj["Size"] | ||
total_files += 1 | ||
|
||
total_size_gb = total_size / (1024**3) # Convert bytes to gigabytes | ||
return {"total_files": total_files, "total_size_gb": total_size_gb} | ||
|
||
except ClientError as e: | ||
print(f"Error getting bucket stats: {e}") | ||
return None | ||
|
||
|
||
class StorageMaintainer(threading.Thread): | ||
"""Maintain frigates recording storage.""" | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
import datetime as dt | ||
import peewee as pw | ||
from playhouse.sqlite_ext import * | ||
from decimal import ROUND_HALF_EVEN | ||
from frigate.models import Recordings | ||
|
||
|
||
def migrate(migrator, database, fake=False, **kwargs): | ||
migrator.add_fields( | ||
Recordings, | ||
storage=pw.CharField(max_length=20,default="local"), | ||
) | ||
|
||
|
||
def rollback(migrator, database, fake=False, **kwargs): | ||
migrator.remove_fields(Recordings, ["storage"]) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,4 @@ | ||
boto3 == 1.26.* | ||
click == 8.1.* | ||
Flask == 2.3.* | ||
imutils == 0.5.* | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Shouldn't we use ssl by default here?