-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add spark outputs, intellij artefact build output to gitignore;
add command example to build sbt; add partitioning, preprocessing scripts that execute Compressor and Partitioner JARs programmatically; add README explaining how to build Compressor and Partitioner JARs using intellij; use Scala Either to differentiate compression, partitioning of weighted vs. unweighted edge lists; add compression tests; Partitioning tests remain to be done; add test resources for CompressorTest;
- Loading branch information
Showing
30 changed files
with
26,901 additions
and
449 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -85,3 +85,9 @@ project/**/metals.sbt | |
| /sandbox/ | ||
| /logs/* | ||
| tmp | ||
|
|
||
| _SUCCESS | ||
| ._SUCCESS.crc | ||
| MANIFEST.MF | ||
| .part-*.crc | ||
| part-* | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| #!/bin/bash | ||
|
|
||
| function parse_yaml { | ||
| local prefix=$2 | ||
| local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034') | ||
| sed -ne "s|^\($s\):|\1|" \ | ||
| -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p" \ | ||
| -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 | | ||
| awk -F$fs '{ | ||
| indent = length($1)/2; | ||
| vname[indent] = $2; | ||
| for (i in vname) {if (i > indent) {delete vname[i]}} | ||
| if (length($3) > 0) { | ||
| vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")} | ||
| printf("%s%s%s=\"%s\"\n", "'$prefix'",vn, $2, $3); | ||
| } | ||
| }' | ||
| } | ||
|
|
||
| akka_gps_home="/home/atrostan/Workspace/repos/akka-gps" | ||
| partitionDriverJarPath="${akka_gps_home}/out/artifacts/akka_gps_jar/akka-gps.jar" | ||
| graphDir="${akka_gps_home}/src/main/resources/graphs/email-Eu-core" | ||
| graphPath="\"${graphDir}/orig.net\"" | ||
| graphYaml="${graphDir}/stats.yml" | ||
| outputPartitionsPath="\"${graphDir}/partitions\"" | ||
| sep="\" \"" | ||
| threshold=100 | ||
| numPartitions=4 | ||
| partitioners=( | ||
| 1 # 1d | ||
| 2 # 2d | ||
| 3 # Hybrid | ||
| ) | ||
| partitionBys=( | ||
| "\"true\"" # partition by source | ||
| "\"false\"" # partition by destination | ||
| ) | ||
| eval $(parse_yaml $graphYaml) | ||
|
|
||
| # programmatically partition an input graph | ||
| for partitioner in "${partitioners[@]}"; do | ||
| for partitionBySource in "${partitionBys[@]}"; do | ||
| javaJarStr="java -jar ${partitionDriverJarPath} --nNodes ${Nodes} --nEdges ${Edges} --inputFilename ${graphPath} --outputDirectoryName ${outputPartitionsPath} --sep ${sep} --partitioner ${partitioner} --threshold ${threshold} --numPartitions ${numPartitions} --partitionBySource ${partitionBySource}" | ||
| echo ${javaJarStr} | ||
| # eval ${javaJarStr} | ||
| done | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| #!/bin/bash | ||
|
|
||
| helpFunction() | ||
| { | ||
| echo "" | ||
| echo "Usage: $0 -c compressParam -p partitionParam" | ||
| echo -e "\t-c 1 or 0. Whether to compress or not" | ||
| echo -e "\t-p 1 or 0. Whether to partition or not" | ||
| exit 1 # Exit script after printing help | ||
| } | ||
|
|
||
| function parse_yaml | ||
| { | ||
| local prefix=$2 | ||
| local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034') | ||
| sed -ne "s|^\($s\):|\1|" \ | ||
| -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p" \ | ||
| -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 | | ||
| awk -F$fs '{ | ||
| indent = length($1)/2; | ||
| vname[indent] = $2; | ||
| for (i in vname) {if (i > indent) {delete vname[i]}} | ||
| if (length($3) > 0) { | ||
| vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")} | ||
| printf("%s%s%s=\"%s\"\n", "'$prefix'",vn, $2, $3); | ||
| } | ||
| }' | ||
| } | ||
|
|
||
| while getopts "c:p:" opt | ||
| do | ||
| case "$opt" in | ||
| c ) compressParam="$OPTARG" ;; | ||
| p ) partitionParam="$OPTARG" ;; | ||
| ? ) helpFunction ;; # Print helpFunction in case parameter is non-existent | ||
| esac | ||
| done | ||
|
|
||
| # Print helpFunction in case parameters are empty | ||
| if [ -z "$compressParam" ] || [ -z "$partitionParam" ] | ||
| then | ||
| echo "Some or all of the parameters are empty"; | ||
| helpFunction | ||
| fi | ||
|
|
||
| # Compress a graph | ||
| # Then, partition the graph using all partition algorithms | ||
| akka_gps_home="/home/atrostan/Workspace/repos/akka-gps" | ||
|
|
||
| partitionDriverJarPath="${akka_gps_home}/out/artifacts/akka_gps_partitioner_jar/akka-gps.jar" | ||
| compressorDriverJarPath="${akka_gps_home}/out/artifacts/akka_gps_compressor_jar/akka-gps.jar" | ||
| graphName="email-Eu-core" | ||
| graphDir="${akka_gps_home}/src/main/resources/graphs/${graphName}" | ||
|
|
||
| # original, uncompressed edgelist | ||
| origGraphPath="\"${graphDir}/origWeighted.net\"" | ||
|
|
||
| # directory that will store the compressed edgelist | ||
| compressedDirName="compressed" | ||
| outputFilename="\"${graphDir}/${compressedDirName}\"" | ||
| compressedGraphPath="\"${graphDir}/${compressedDirName}/part-00000\"" | ||
|
|
||
| sep="\" \"" | ||
| isWeighted="\"true\"" | ||
| graphYaml="${graphDir}/stats.yml" | ||
| outputPartitionsPath="\"${graphDir}/partitions\"" | ||
| threshold=100 | ||
| numPartitions=4 | ||
|
|
||
| partitioners=( | ||
| 1 # 1d | ||
| 2 # 2d | ||
| 3 # Hybrid | ||
| ) | ||
|
|
||
| partitionBys=( | ||
| "\"true\"" # partition by source | ||
| "\"false\"" # partition by destination | ||
| ) | ||
|
|
||
| # compress the graph | ||
| compressJavaJarStr="java -jar ${compressorDriverJarPath} --inputFilename ${origGraphPath} --outputFilename ${outputFilename} --sep ${sep} --isWeighted ${isWeighted}" | ||
| # echo ${compressJavaJarStr} | ||
|
|
||
| if [ $compressParam -eq 1 ] | ||
| then | ||
| eval ${compressJavaJarStr} | ||
| fi | ||
| # read nNodes, nEdges from stats.yml | ||
| eval $(parse_yaml $graphYaml) | ||
|
|
||
| if [ $partitionParam -eq 1 ] | ||
| then | ||
| # programmatically partition an input graph | ||
| for partitioner in "${partitioners[@]}"; do | ||
| for partitionBySource in "${partitionBys[@]}"; do | ||
| printf "\n" | ||
| logStr="Partitioning ${compressedGraphPath} with ${Nodes} Nodes and ${Edges} Edges.\nPartitioner: ${partitioner}, Partitioning By Source: ${partitionBySource}, Number of Partitions: ${numPartitions}, Threshold: ${threshold}, isWeighted: ${isWeighted}" | ||
| echo -e ${logStr} | ||
| printf "\n" | ||
|
|
||
| javaJarStr="java -jar ${partitionDriverJarPath} --nNodes ${Nodes} --nEdges ${Edges} --inputFilename ${compressedGraphPath} --outputDirectoryName ${outputPartitionsPath} --sep ${sep} --partitioner ${partitioner} --threshold ${threshold} --numPartitions ${numPartitions} --partitionBySource ${partitionBySource} --isWeighted ${isWeighted}" | ||
| # echo ${javaJarStr} | ||
| eval ${javaJarStr} | ||
| done | ||
| done | ||
| fi |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| Nodes: 8 | ||
| Edges: 32 |
Oops, something went wrong.