Add a seeder

experiment/popularity_community/crawl_torrents.py

@@ -45,13 +45,18 @@
 
 import sentry_sdk
 
-from experiment.tool.tiny_tribler_service import TinyTriblerService
-
-from tribler_core.modules.metadata_store.community.remote_query_community import RemoteQueryCommunity, \
- RemoteSelectPayload, SelectRequest, SelectResponsePayload
+from tribler_core.modules.metadata_store.community.remote_query_community import (
+ RemoteQueryCommunity,
+ RemoteSelectPayload,
+ SelectRequest,
+ SelectResponsePayload,
+)
 from tribler_core.modules.metadata_store.serialization import REGULAR_TORRENT
+from tribler_core.utilities.tiny_tribler_service import TinyTriblerService
 from tribler_core.utilities.unicode import hexlify
 
+# flake8: noqa
+
 UNLIMITED = -1 # const. Don't change.
 
 IPV8_WALK_INTERVAL = 0.05

experiment/popularity_community/initial_filling.py

@@ -14,7 +14,7 @@
 
 import sentry_sdk
 
-from experiment.tool.tiny_tribler_service import TinyTriblerService
+from tribler_core.utilities.tiny_tribler_service import TinyTriblerService
 
 from tribler_core.modules.popularity.popularity_community import PopularityCommunity
 

experiment/tunnel_community/hidden_peer_discovery.py

@@ -6,7 +6,7 @@
 
 from ipv8.taskmanager import TaskManager
 
-from experiment.tool.tiny_tribler_service import TinyTriblerService
+from tribler_core.utilities.tiny_tribler_service import TinyTriblerService
 
 EXPERIMENT_RUN_TIME = int(os.environ.get('EXPERIMENT_RUN_TIME', 3600 * 3))
 

experiment/tunnel_community/speed_test_exit.py

@@ -6,7 +6,7 @@
 from ipv8.messaging.anonymization.utils import run_speed_test
 from ipv8.taskmanager import TaskManager
 
-from experiment.tool.tiny_tribler_service import TinyTriblerService
+from tribler_core.utilities.tiny_tribler_service import TinyTriblerService
 
 EXPERIMENT_NUM_MB = int(os.environ.get('EXPERIMENT_NUM_MB', 25))
 EXPERIMENT_NUM_CIRCUITS = int(os.environ.get('EXPERIMENT_NUM_CIRCUITS', 10))

src/seedbox/README.md

@@ -0,0 +1,97 @@
+# Seedbox
+
+This folder contains scripts for effortlessly setting up a seedbox.
+
+The seedbox consists of two parts:
+
+1. Torrent seeding (by using a LibTorrent protocol)
+1. Channel disseminating (by using the Tribler network)
+
+## Prerequisites
+
+1. Clone the tribler repo include sub modules:
+ ```shell
+ git clone --recursive https://github.com/Tribler/tribler.git
+ ```
+1. Install requirements:
+ ```bash
+ python3 -m pip install -r requirements.txt 
+ ```
+1. Add necessary folders to `PYTHONPATH` (below the bash example)
+ ```shell
+ export PYTHONPATH=${PYTHONPATH}:`echo ../.. ../{pyipv8,tribler-common,tribler-core} | tr " " :`
+ ```
+
+## Torrent seeding
+
+To start torrents' seeding run the following script:
+
+```bash
+python3 seeder.py <source folder>
+```
+
+Consider the following folder structure:
+
+```
+source folder
+├ sub_directory
+| ├ file1
+| └file2
+├ sub_directory2
+| ├ file3
+| └ file4
+├ thumbnail.png
+└ description.md
+```
+
+In this particular example, `seeder.py` will create two torrents:
+`sub_directory.torrent` and `sub_directory2.torrent`.
+
+`seeder.py` will start to seed them through BitTorrent protocol after creating.
+
+## Data disseminating
+
+To start disseminating data through Tribler's network run the following script:
+
+```bash
+python3 disseminator.py <source folder>
+```
+
+This script will create a channel and will disseminate it to Tribler.
+
+Consider the following folder structure:
+
+```
+source folder
+├ sub_directory.torrent
+├ sub_directory2.torrent
+├ thumbnail.png
+└ description.md
+```
+
+Above you can see two "special" files:
+* thumbnail.png
+* description.md
+
+The channel will be created with description based on these files. 
+As the channel name, the source folder's name will be used.
+
+### Error reporting
+
+In case you want errors to be reported, you can use [Sentry](https://develop.sentry.dev/)
+
+To enable error reporting, specify the following environment variable:
+
+```bash
+export SENTRY_URL=<sentry_url>
+```
+
+URL can be taken directly from a corresponding Sentry project.
+
+### Generate test data
+
+The following script generates `1GB` dataset divided into `1024` folders:
+
+```shell
+python3 generate_test_data.py -d /tmp/test_data 
+```

src/seedbox/disseminator.py

@@ -0,0 +1,234 @@
+"""
+This script scans the input directory and create a Tribler channel based on
+torrents found.
+
+For available parameters see "parse_args" function below.
+
+Folder structure:
+
+my channel
+├ sub_directory
+| ├ file1.torrent
+| └ file2.torrent
+├ file3.torrent
+├ thumbnail.png
+└ description.md
+"""
+
+import argparse
+import asyncio
+import logging
+import os
+from json import dumps
+from pathlib import Path
+from types import SimpleNamespace
+
+import libtorrent
+
+from pony.orm import db_session
+
+import sentry_sdk
+
+from tribler_core.modules.libtorrent.torrentdef import TorrentDef
+from tribler_core.modules.metadata_store.community.gigachannel_community import GigaChannelCommunity
+from tribler_core.modules.metadata_store.orm_bindings.channel_node import NEW
+from tribler_core.utilities.tiny_tribler_service import TinyTriblerService
+
+# fmt: off
+# flake8: noqa
+
+
+_description_file_name = 'description.md'
+_thumbnail_file_name = 'thumbnail.png'
+
+_logger = logging.getLogger('Disseminator')
+
+sentry_sdk.init(
+ os.environ.get('SENTRY_URL'),
+ traces_sample_rate=1.0
+)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Disseminate data by using the Tribler network')
+
+ parser.add_argument('-s', '--source', type=str, help='path to data folder', default='.')
+ parser.add_argument('-t', '--tribler_dir', type=str, help='path to data folder', default='$HOME/seedbox/.Tribler')
+ parser.add_argument('-v', '--verbosity', help='increase output verbosity', action='store_true')
+
+ return parser.parse_args()
+
+
+def setup_logger(verbosity):
+ logging_level = logging.DEBUG if verbosity else logging.INFO
+ logging.basicConfig(level=logging_level)
+
+
+class ChannelHelper:
+ def __init__(self, community, manager):
+ self.community = community
+ self.manager = manager
+ self.directories = SimpleNamespace(tree={}, directory=None)
+
+ @db_session
+ def create_root_channel(self, name, description=''):
+ _logger.info(f'Creating channel: {name}')
+ channels = self.community.mds.ChannelMetadata
+
+ if len(channels.get_channels_by_title(name)) >= 1:
+ _logger.warning(f'Channel with name {name} already exists')
+ return False
+
+ self.directories.directory = channels.create_channel(name, description)
+ self.flush()
+
+ return True
+
+ @db_session
+ def add_torrent(self, file, relative_path):
+ _logger.info(f'Add torrent: {file}')
+
+ directory = self.get_directory(relative_path)
+ decoded_torrent = libtorrent.bdecode(file.read_bytes())
+ directory.add_torrent_to_channel(TorrentDef(metainfo=decoded_torrent), None)
+
+ @db_session
+ def add_thumbnail(self, thumbnail):
+ if not thumbnail:
+ return
+
+ _logger.info(f'Add thumbnail: {thumbnail}')
+
+ root_channel = self.directories.directory
+ self.community.mds.ChannelThumbnail(public_key=root_channel.public_key,
+ origin_id=root_channel.id_,
+ status=NEW,
+ binary_data=thumbnail,
+ data_type='image/png')
+
+ @db_session
+ def add_description(self, description):
+ if not description:
+ return
+
+ _logger.info(f'Add description: {description}')
+
+ root_channel = self.directories.directory
+ self.community.mds.ChannelDescription(public_key=root_channel.public_key,
+ origin_id=root_channel.id_,
+ json_text=dumps({"description_text": description}),
+ status=NEW)
+
+ @db_session
+ def get_directory(self, path):
+ current = self.directories
+
+ for part in path.parts[:-1]:
+ next_directory = current.tree.get(part, None)
+ if next_directory is not None:
+ current = next_directory
+ continue
+
+ next_directory = SimpleNamespace(
+ tree={},
+ directory=self.community.mds.CollectionNode(title=part, origin_id=current.directory.id_, status=NEW)
+ )
+
+ current.tree[part] = next_directory
+ current = next_directory
+ self.flush()
+
+ _logger.info(f'Directory created: {part}')
+
+ return current.directory
+
+ @db_session
+ def commit(self):
+ _logger.info('Commit changes')
+
+ for t in self.community.mds.CollectionNode.commit_all_channels():
+ self.manager.updated_my_channel(TorrentDef.load_from_dict(t))
+
+ @db_session
+ def flush(self):
+ _logger.debug('Flush')
+
+ self.community.mds._db.flush() # pylint: disable=protected-access
+
+
+class Service(TinyTriblerService):
+ def __init__(self, source_dir, working_dir, config_path):
+ super().__init__(Service.create_config(working_dir, config_path),
+ working_dir=working_dir,
+ config_path=config_path)
+ self.source_dir = Path(source_dir)
+
+ @staticmethod
+ def create_config(working_dir, config_path):
+ config = TinyTriblerService.create_default_config(working_dir, config_path)
+
+ config.set_libtorrent_enabled(True)
+ config.set_ipv8_enabled(True)
+ config.set_chant_enabled(True)
+ config.set_chant_manager_enabled(True)
+
+ return config
+
+ def get_torrents_from_source(self):
+ return [(file, file.relative_to(self.source_dir)) for file in self.source_dir.rglob('*.torrent')]
+
+ def get_thumbnail(self):
+ file = self.source_dir / _thumbnail_file_name
+ return file.read_bytes() if file.exists() else None
+
+ def get_description(self):
+ file = self.source_dir / _description_file_name
+ return file.read_text() if file.exists() else None
+
+ async def create_channel(self, community, manager):
+ channel_helper = ChannelHelper(community, manager)
+ channel_name = self.source_dir.name
+
+ if not channel_helper.create_root_channel(channel_name):
+ return
+
+ torrents = self.get_torrents_from_source()
+
+ for file, relative_path in torrents:
+ channel_helper.add_torrent(file, relative_path)
+
+ channel_helper.add_thumbnail(self.get_thumbnail())
+ channel_helper.add_description(self.get_description())
+
+ channel_helper.commit()
+
+ _logger.info(f'{len(torrents)} torrents where added')
+
+ async def on_tribler_started(self):
+ await super().on_tribler_started()
+ await self.create_channel(self.session.ipv8.get_overlay(GigaChannelCommunity),
+ self.session.gigachannel_manager)
+
+
+def run_tribler(arguments):
+ service = Service(
+ source_dir=Path(arguments.source),
+ working_dir=Path(arguments.tribler_dir),
+ config_path=Path('./tribler.conf')
+ )
+
+ loop = asyncio.get_event_loop()
+ loop.create_task(service.start_tribler())
+ try:
+ loop.run_forever()
+ finally:
+ loop.run_until_complete(loop.shutdown_asyncgens())
+ loop.close()
+
+
+if __name__ == "__main__":
+ _arguments = parse_args()
+ print(f"Arguments: {_arguments}")
+
+ setup_logger(_arguments.verbosity)
+ run_tribler(_arguments)