morpheus.code-workspace

{
	"folders": [
		{
			"path": ".",
		}
	],
	"settings": {
		"[cpp]": {
			"editor.wordBasedSuggestions": false,
			"editor.suggest.insertMode": "replace",
			"editor.semanticHighlighting.enabled": true,
			"editor.tabSize": 4,
			"editor.detectIndentation": false,
			"editor.wordWrapColumn": 120,
			"editor.formatOnSave": true,
		},
		"[python]": {
			"editor.codeActionsOnSave": {
				"source.organizeImports": true
			},
			"editor.formatOnSave": true,
			"editor.rulers": [
				120
			],
			"editor.tabSize": 4
		},
		"[rst]": {
			"editor.rulers": [
				120
			]
		},
		"cmake.configureArgs": [
			"-DCMAKE_MESSAGE_CONTEXT_SHOW=ON", // Show message context by default
			"-DMORPHEUS_USE_CLANG_TIDY=OFF", // If we are using clangd extension, disable clang-tidy in VS Code to prevent double errors
			"-DMORPHEUS_PYTHON_INPLACE_BUILD=ON", // Allow inplace build for python. Use `pip install -e .` from the python folder to install
			"-DMORPHEUS_USE_CCACHE=ON", // Enable ccache when building in vscode
		],
		"files.insertFinalNewline": true,
		"files.trimFinalNewlines": true,
		"files.trimTrailingWhitespace": true,
		"python.formatting.provider": "yapf",
		"python.formatting.yapfArgs": [
			"--style=${workspaceFolder}/setup.cfg"
		],
		"python.linting.flake8Enabled": true,
		"python.linting.pylintArgs": [
			"--rcfile=${workspaceFolder}/.pylintrc",
			"--init-hook=import sys; sys.path.append(\"${workspaceFolder}\")",
		],
		"python.linting.pylintEnabled": true,
		"rewrap.wrappingColumn": 120,
		"python.testing.pytestArgs": [
			"-s",
			"tests"
		],
		"python.testing.unittestEnabled": false,
		"python.testing.pytestEnabled": true
	},
	"launch": {
		"configurations": [
			{
				"name": "Python: Run Pipeline (NLP)",
				"type": "python",
				"request": "launch",
				"program": "${workspaceFolder}/morpheus/cli/run.py",
				"console": "integratedTerminal",
				"justMyCode": false,
				"subProcess": true,
				"cwd": "${workspaceFolder}",
				"args": [
					"--log_level=DEBUG",
					// "--debug",
					"run",
					"--num_threads=8",
					"--pipeline_batch_size=1024",
					"--model_max_batch_size=32",
					"--edge_buffer_size=4",
					"--use_cpp=False",
					"pipeline-nlp",
					"--model_seq_length=256",
					"from-file",
					"--filename=models/datasets/validation-data/sid-validation-data.csv",
					"--repeat=50",
					// "dropna",
					// "from-kafka",
					"deserialize",
					"preprocess",
					"--vocab_hash_file=./morpheus/data/bert-base-uncased-hash.txt",
					"--truncation=True",
					"--do_lower_case=True",
					"--add_special_tokens=False",
					"inf-triton",
					"--model_name=sid-minibert-onnx",
					"--server_url=localhost:8001",
					"--force_convert_inputs=True",
					// "--use_shared_memory=True",
					"monitor",
					"--description",
					"Inference Rate",
					"--unit",
					"inf",
					"add-class",
					// "filter",
					"serialize",
					// "--include",
					// "timestamp",
					"--exclude",
					"^_ts_",
					"--output_type=json",
					// "to-kafka",
					// "--output_topic",
					// "inference_output",
					"to-file",
					"--filename=./.tmp/sid_training_data-nlp.csv",
					"--overwrite",
				]
			},
			{
				"name": "Python: Run Pipeline (FIL)",
				"type": "python",
				"request": "launch",
				"program": "${workspaceFolder}/morpheus/cli/run.py",
				"console": "integratedTerminal",
				"justMyCode": false,
				"subProcess": true,
				"cwd": "${workspaceFolder}",
				"args": [
					"--log_level=DEBUG",
					// "--debug",
					"run",
					"--num_threads=8",
					"--pipeline_batch_size=1024",
					"--model_max_batch_size=1024",
					// "--use_cpp=False",
					"pipeline-fil",
					"from-file",
					"--filename=./models/datasets/validation-data/abp-validation-data.jsonlines",
					"--repeat=200",
					// "from-kafka",
					// "dropna",
					"deserialize",
					"preprocess",
					"inf-triton",
					"--model_name=abp-nvsmi-xgb",
					"--server_url=localhost:8001",
					"--force_convert_inputs=True",
					"monitor",
					"--description",
					"Inference Rate",
					"--smoothing=0.001",
					"--unit",
					"inf",
					"add-class",
					// "validate",
					// "--val_file_name=models/datasets/validation-data/abp-validation-data.jsonlines",
					// "--results_file_name=./.tmp/validation_results-fil.json",
					// "--overwrite",
					"filter",
					"serialize",
					"--include",
					"timestamp",
					"--exclude",
					"^_ts_",
					"--exclude",
					"^nvidia_smi_log",
					// "to-kafka",
					// "--output_topic",
					// "inference_output",
					"to-file",
					"--filename=./.tmp/abp-validation-data.csv",
					"--overwrite",
				]
			},
			{
				"name": "Python: Run Pipeline (AE)",
				"type": "python",
				"request": "launch",
				"program": "${workspaceFolder}/morpheus/cli/run.py",
				"console": "integratedTerminal",
				"justMyCode": false,
				"subProcess": true,
				"cwd": "${workspaceFolder}",
				"args": [
					"--log_level=DEBUG",
					"run",
					"--num_threads=1",
					"--pipeline_batch_size=1024",
					"--model_max_batch_size=1024",
					"--use_cpp=False",
					"pipeline-ae",
                    "--userid_column_name=userIdentitysessionContextsessionIssueruserName",
					"--userid_filter=user123",
					"from-cloudtrail",
					"--input_glob=models/datasets/validation-data/dfp-cloudtrail-*-input.csv",
					"--max_files=200",
					"train-ae",
					"--train_data_glob=models/datasets/training-data/dfp-*.csv",
					"--seed=42",
					"preprocess",
					"inf-pytorch",
					"add-scores",
					"timeseries",
					"--resolution=1m",
					"--zscore_threshold=8.0",
					"--hot_start",
					"monitor",
					"--description",
					"Inference Rate",
					"--unit",
					"inf",
					"validate",
					"--val_file_name=models/datasets/validation-data/dfp-cloudtrail-user123-validation-data-output.csv",
					"--results_file_name=./.tmp/validation_results-ae.json",
					"--index_col=_index_",
					"--exclude=event_dt",
					"--rel_tol=0.1",
					"--overwrite",
					"serialize",
					// "--include",
					// "timestamp",
					// "--exclude",
					// "^_ts_",
					// "--exclude",
					// "^nvidia_smi_log",
					// "to-kafka",
					// "--output_topic",
					// "inference_output",
					"to-file",
					"--filename=./.tmp/detections.csv",
					"--overwrite",
				]
			},
			{
				"name": "Python: Run Pipeline (NLP-Phishing)",
				"type": "python",
				"request": "launch",
				"program": "${workspaceFolder}/morpheus/cli/run.py",
				"console": "integratedTerminal",
				"justMyCode": false,
				"subProcess": true,
				"cwd": "${workspaceFolder}",
				"args": [
					"--log_level=DEBUG",
					// "--debug",
					"run",
					"--num_threads=1",
					"--pipeline_batch_size=1024",
					"--model_max_batch_size=32",
					// "--use_cpp=False",
					"pipeline-nlp",
					"--labels_file=morpheus/data/labels_phishing.txt",
					"--model_seq_length=128",
					"from-file",
					"--filename=models/datasets/validation-data/phishing-email-validation-data.jsonlines",
					"--repeat=1",
					// "dropna",
					// "from-kafka",
					"deserialize",
					"preprocess",
					"--vocab_hash_file=./morpheus/data/bert-base-uncased-hash.txt",
					"--truncation=True",
					// "--stride=",
					"--do_lower_case=True",
					"--add_special_tokens=False",
					"inf-triton",
					"--model_name=phishing-bert-onnx",
					"--server_url=localhost:8001",
					"--force_convert_inputs=True",
					// "--use_shared_memory=True",
					"monitor",
					"--description",
					"Inference Rate",
					"--unit",
					"inf",
					"add-class",
					// "--label=pred",
					// "--threshold=0.7",
					// "add-scores",
					// "--label=score",
					// "filter",
					"validate",
					"--val_file_name=models/datasets/validation-data/phishing-email-validation-data.jsonlines",
					"--results_file_name=./.tmp/validation_results-phishing.json",
					"--overwrite",
					"serialize",
					// "--include",
					// "timestamp",
					"--exclude",
					"^ts_",
					// "to-kafka",
					// "--output_topic",
					// "inference_output",
					"to-file",
					"--filename=./.tmp/phishing-detections.jsonlines",
					"--overwrite",
				]
			},
			{
				"name": "Python: Anomaly Detection Example",
				"type": "python",
				"request": "launch",
				"program": "${workspaceFolder}/examples/abp_pcap_detection/run.py",
				"console": "integratedTerminal",
				"cwd": "${workspaceFolder}/examples/abp_pcap_detection",
				"justMyCode": false,
				"args": [
					"--input_file",
					"../data/pcap_dump.jsonlines",
					"--output_file",
					"./.tmp/pcap_output.jsonlines",
					"--server_url",
					"localhost:8001",
					"--model_name",
					"abp-pcap-xgb"
				]
			},
			{
				"name": "Python: Sphinx",
				"type": "python",
				"request": "launch",
				"module": "sphinx.cmd.build",
				"console": "integratedTerminal",
				"cwd": "${workspaceFolder}/docs",
				"args": [
					"-b",
					"html",
					"-aE",
					"source",
					"build",
				],
				"justMyCode": false
			},
			{
				"name": "Debug MRC from Python (Morpheus-NLP)",
				"type": "cppdbg",
				"request": "launch",
				"program": "python",
				"args": [
					"./morpheus/cli.py",
					"--log_level=DEBUG",
					"run",
					"--num_threads=2",
					"--pipeline_batch_size=1024",
					"--model_max_batch_size=32",
					"--edge_buffer_size=4",
					"--use_cpp=True",
					"pipeline-nlp",
					"--model_seq_length=256",
					// "--viz_file=.tmp/nlp-viz.png",
					"from-file",
					"--filename=models/datasets/validation-data/sid-validation-data.csv",
					"--repeat=5",
					//  "--iterative",
					// "from-kafka",
					"deserialize",
					"preprocess",
					"--vocab_hash_file=./morpheus/data/bert-base-uncased-hash.txt",
					"--truncation=True",
					"--do_lower_case=True",
					"--add_special_tokens=False",
					"inf-triton",
					"--model_name=sid-minibert-onnx",
					"--server_url=localhost:8001",
					"--force_convert_inputs=True",
					"monitor",
					"--description",
					"Inference Rate",
					"--unit",
					"inf",
					"add-class",
					// "filter",
					"serialize",
					// "--include",
					// "timestamp",
					"--exclude",
					"^_ts_",
					// "to-kafka",
					// "--output_topic",
					// "inference_output",
					"to-file",
					"--filename=./.tmp/detections2-nlp.csv",
					"--overwrite",
				],
				"stopAtEntry": false,
				"cwd": "${workspaceFolder}",
				"environment": [
					{
						"name": "MORPHEUS_ROOT",
						"value": "${workspaceFolder}"
					},
					{
						"name": "GLOG_v",
						"value": "10"
					},
					{
						"name": "GLOG_logtostderr",
						"value": "1"
					},
					{
						"name": "CUDA_LAUNCH_BLOCKING",
						"value": "1"
					}
				],
				"externalConsole": false,
				"MIMode": "gdb",
				"setupCommands": [
					{
						"description": "Enable pretty-printing for gdb",
						"text": "-enable-pretty-printing",
						"ignoreFailures": true
					},
					{
						"description": "Skip stdio-common files",
						"text": "-interpreter-exec console \"skip -gfi **/bits/*.h\""
					},
				],
				"symbolLoadInfo": {
					"loadAll": false,
					"exceptionList": "libmrc*.so;cudf_helpers.*;executor.*;morpheus.*;node.*;options.*;pipeline.*;segment.*;subscriber.*;stages.*;messages.*;common*.so"
				},
				"miDebuggerPath": "gdb",
				"sourceFileMap": {
					"${workspaceFolder}": {
						"editorPath": "${workspaceFolder}",
						"useForBreakpoints": "true"
					},
				},
			},
			{
				"name": "Debug MRC from Python (Morpheus-FIL)",
				"type": "cppdbg",
				"request": "launch",
				"program": "python",
				"args": [
					"./morpheus/cli.py",
					"--log_level=DEBUG",
					// "--debug",
					"run",
					"--num_threads=1",
					"--pipeline_batch_size=1024",
					"--model_max_batch_size=1024",
					// "--use_cpp=False",
					"pipeline-fil",
					"from-file",
					"--filename=models/datasets/validation-data/abp-validation-data.csv",
					"--repeat=10",
					"--iterative",
					// "from-kafka",
					// "dropna",
					"deserialize",
					"preprocess",
					"inf-triton",
					"--model_name=abp-nvsmi-xgb",
					"--server_url=localhost:8001",
					"--force_convert_inputs=True",
					"monitor",
					"--description",
					"Inference Rate",
					"--unit",
					"inf",
					"add-class",
					"validate",
					"--val_file_name=models/datasets/validation-data/abp-validation-data.jsonlines",
					"--results_file_name=./.tmp/validation_results-fil.json",
					"--overwrite",
					// "filter",
					"serialize",
					// "--include",
					// "timestamp",
					"--exclude",
					"^_ts_",
					// "--exclude",
					// "^nvidia_smi_log",
					// "to-kafka",
					// "--output_topic",
					// "inference_output",
					"to-file",
					"--filename=./.tmp/abp-validation-data.csv",
					"--overwrite",
				],
				"stopAtEntry": false,
				"cwd": "${workspaceFolder}",
				"environment": [
					{
						"name": "MORPHEUS_ROOT",
						"value": "${workspaceFolder}"
					},
					{
						"name": "GLOG_v",
						"value": "10"
					},
					{
						"name": "CUDA_LAUNCH_BLOCKING",
						"value": "1"
					}
				],
				"externalConsole": false,
				"MIMode": "gdb",
				"setupCommands": [
					{
						"description": "Enable pretty-printing for gdb",
						"text": "-enable-pretty-printing",
						"ignoreFailures": true
					},
					{
						"description": "Skip stdio-common files",
						"text": "-interpreter-exec console \"skip -gfi **/bits/*.h\""
					},
				],
				"symbolLoadInfo": {
					"loadAll": false,
					"exceptionList": "libmrc*.so;cudf_helpers.*;executor.*;morpheus.*;node.*;options.*;pipeline.*;segment.*;subscriber.*;stages.*;messages.*;common*.so"
				},
				"miDebuggerPath": "gdb",
				"sourceFileMap": {
					"${workspaceFolder}": {
						"editorPath": "${workspaceFolder}",
						"useForBreakpoints": "true"
					}
				},
			},
			{
				"name": "Debug MRC from Python (Morpheus-AE)",
				"type": "cppdbg",
				"request": "launch",
				"program": "python",
				"args": [
					"./morpheus/cli.py",
					"--log_level=DEBUG",
					"run",
					"--num_threads=1",
					"--pipeline_batch_size=128",
					"--model_max_batch_size=128",
					"--use_cpp=False",
					"pipeline-ae",
					// "--ae_path=models/hammah-models/hammah-role-g-20211017.pkl",
					"--ae_path=../data/ae_model.pkl",
					"from-cloudtrail",
					"--input_glob=models/datasets/validation-data/dfp-cloudtrail-role-g-validation-data.csv",
					// "--input_glob=./data/red_team.csv",
					"--max_files=200",
					"--iterative",
					"deserialize",
					"preprocess",
					"inf-triton",
					"--model_name=autoencoder-onnx",
					"--server_url=localhost:8001",
					"timeseries",
					"--resolution=10m",
					"monitor",
					"--description",
					"Inference Rate",
					"--smoothing=0.001",
					"--unit",
					"inf",
					// "add-class",
					// "filter",
					// "serialize",
					// "--include",
					// "timestamp",
					// "--exclude",
					// "^_ts_",
					// "--exclude",
					// "^nvidia_smi_log",
					// "to-kafka",
					// "--output_topic",
					// "inference_output",
					// "to-file",
					// "--filename=./.tmp/detections.json",
					// "--overwrite",
				],
				"stopAtEntry": false,
				"cwd": "${workspaceFolder}",
				"environment": [
					{
						"name": "MORPHEUS_ROOT",
						"value": "${workspaceFolder}"
					},
					{
						"name": "GLOG_v",
						"value": "10"
					},
					{
						"name": "CUDA_LAUNCH_BLOCKING",
						"value": "1"
					}
				],
				"externalConsole": false,
				"MIMode": "gdb",
				"setupCommands": [
					{
						"description": "Enable pretty-printing for gdb",
						"text": "-enable-pretty-printing",
						"ignoreFailures": true
					},
					{
						"description": "Skip stdio-common files",
						"text": "-interpreter-exec console \"skip -gfi **/bits/*.h\""
					},
				],
				"symbolLoadInfo": {
					"loadAll": false,
					"exceptionList": "libmrc*.so;cudf_helpers.*;executor.*;morpheus.*;node.*;options.*;pipeline.*;segment.*;subscriber.*;stages.*;messages.*;common*.so"
				},
				"miDebuggerPath": "gdb",
				"sourceFileMap": {
					"${workspaceFolder}": {
						"editorPath": "${workspaceFolder}",
						"useForBreakpoints": "true"
					}
				},
			},
		],
		"compounds": []
	}
}