Skip to content

Commit

Permalink
changes for updated redcap-etl
Browse files Browse the repository at this point in the history
  • Loading branch information
@naalong
naalong committed May 11, 2021
1 parent b7cd34d commit 95496ee
Show file tree
Hide file tree
Showing 706 changed files with 966 additions and 283,348 deletions.
8 changes: 8 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,11 @@ test-config.ini
# Ignore web test coverage data files, except for README
/tests/web/coverage-data/*
!/tests/web/coverage-data/README.md


# Ignore dependencies directory
/dependencies

# Composer lock file
composer.lock
composer.*
213 changes: 145 additions & 68 deletions RedCapEtlModule.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1052,7 +1052,7 @@ public function renderUserTabs($activeUrl = '')
{
$listUrl = $this->getUrl('web/index.php');
$listLabel = '<span class="fas fa-list"></span>'
. ' ETL Configurations';
. ' ETL Tasks';


$configUrl = $this->getUrl('web/configure.php');
Expand All @@ -1070,7 +1070,7 @@ public function renderUserTabs($activeUrl = '')

$workflowsUrl = $this->getUrl('web/workflows.php');
$workflowsLabel = '<span style="color: #808080;" class="fas fa-list"></span>'
.' Workflows';
.' ETL Workflows';

$userManualUrl = $this->getUrl('web/user_manual.php');
$userManualLabel =
Expand Down Expand Up @@ -1463,6 +1463,8 @@ public function setWorkflowGlobalProperties($workflowName, $properties, $usernam
public function runWorkflow(
$workflowName,
$serverName,
$username,
$userProjects,
$isCronJob = false,
$originatingProjectId = PROJECT_ID,
$cronJobLogId = null,
Expand All @@ -1473,11 +1475,13 @@ public function runWorkflow(
$status = null;

try {
$username = '';
if (defined('USERID')) {
$username = USERID;
if (empty($usernamei)) {
if (defined('USERID')) {
$username = USERID;
}
}

$superUser = SUPER_USER;
#-------------------------------------------------
# Log run information to external module log
#-------------------------------------------------
Expand Down Expand Up @@ -1529,86 +1533,156 @@ public function runWorkflow(
if (empty($serverName)) {
throw new \Exception('No server specified.');
} else {
$serverConfig = $this->getServerConfig($serverName); # Method throws exception if server not found
$serverConfig = $this->getServerConfig($serverName); # calls settings->getserverconfig
if (!$serverConfig->getIsActive()) {
throw new \Exception('Server "'.$serverName.'" has been deactivated and cannot be used.');
}
}

$cron = 'no';
if ($isCronJob) {
$cron = 'yes';
}

$workflowProperties = array();
$workflowProperties['workflow_name'] = $workflowName;

#---------------------------------------------
# Add global variables to workflow properties
#---------------------------------------------
$workflowProperties = array_merge($workflowProperties, array_filter($globalProperties));

#---------------------------------------------
# Run each workflow ETL task
#---------------------------------------------
$isWorkflow = true;
#$workflowProperties['username'] = $username;
#$workflowProperties['serverName'] = $serverName;
#$workflowProperties['tasks'] = array();
foreach ($tasks as $key => $task) {
$projectId = $task['projectId'];
$configName = $task['projectEtlConfig'];
if (!empty($configName)) {
Configuration::validateName($configName); # throw exception if invalid
$etlConfig = $this->getConfiguration($configName, $projectId);
if (isset($etlConfig)) {
$etlProperties = $etlConfig->getProperties();
} else {
throw new \Exception('Configuration "'.$configName.'" not found for project ID '.$projectId);
}

#check to see if the user has permissions to export for this project
$hasPermissionToExport = false;
if ($superUser) {
$hasPermissionToExport = true;
} else {
throw new \Exception('No configuration name provided for project ID '.$projectId);
$pKey = array_search($projectId, array_column($userProjects, 'project_id'));
if ($pKey || $pKey === 0) {
$hasPermissionToExport = $userProjects[$pKey]['data_export_tool'] == 1 ? true : false;
}
}

$taskProperties = $this->getWorkflowTaskProperties($workflowName, $globalProperties, $etlProperties);
$isWorkflow = true;
$etlConfig->set($taskProperties, $isWorkflow);
if ($hasPermissionToExport) {
$projectId = $task['projectId'];
$taskName = $task['taskName'];
#$workflowProperties['tasks'][$taskName] = array();
$workflowProperties[$taskName] = array();

$configName = $task['projectEtlConfig'];
if (!empty($configName)) {
Configuration::validateName($configName); # throw exception if invalid
$etlConfig = $this->getConfiguration($configName, $projectId);
if (isset($etlConfig)) {
$etlProperties = $etlConfig->getProperties();
} else {
throw new \Exception('For Workflow "'.$workflowName.'", Configuration "'.$configName.'" not found for project ID '.$projectId);
}
} else {
throw new \Exception('No configuration name provided for Workflow "'.$workflowName.'", project ID '.$projectId);
}

$configUrl = $etlConfig->getProperty(Configuration::REDCAP_API_URL);
#get the task properties updated with the global properties
#$taskProperties = $this->getWorkflowTaskProperties($workflowName, $globalProperties, $etlProperties);

#update the task configuration with the task properties
#$etlConfig->set($taskProperties, $isWorkflow);

#update the task configuration with server values
$serverConfig->updateEtlConfig($etlConfig, $isCronJob);
#print "=================== Redcapetlmodule.php, runWorkflow, 1596, etlConfig is ";
#print_r($etlConfig);
# $etlConfig->validateForRunning($isWorkflow);
$workflowProperties[$taskName] = $etlConfigi['properties'];
print "=================== Redcapetlmodule.php, runWorkflow, 1600, workflowProperties is ";
print_r($workflowProperties);
# YOU ARE HERE, see what the workflow properties look like
$configUrl = $etlConfig->getProperty(Configuration::REDCAP_API_URL);

#-----------------------------------------------------
# Set task logging information
#-----------------------------------------------------
$details = '';
# If being run on remote REDCap
if (strcasecmp($configUrl, $this->getRedCapApiUrl()) !== 0) {
$details .= "REDCap API URL: {$configUrl}\n";
} else {
# If local REDCap is being used, set SSL verify to the global value
$sslVerify = $adminConfig->getSslVerify();
$etlConfig->setProperty(Configuration::SSL_VERIFY, $sslVerify);
}
#-----------------------------------------------------
# Set task logging information
#-----------------------------------------------------
$details = '';
# If being run on remote REDCap
if (strcasecmp($configUrl, $this->getRedCapApiUrl()) !== 0) {
$details .= "REDCap API URL: {$configUrl}\n";
} else {
# If local REDCap is being used, set SSL verify to the global value
$sslVerify = $adminConfig->getSslVerify();
$etlConfig->setProperty(Configuration::SSL_VERIFY, $sslVerify);
}

$details .= "project ID: {$projectId}\n";
if (!$isCronJob) {
$details .= "user: ".USERID."\n";
}
$details .= "configuration: {$configName}\n";
$details .= "server: {$serverName}\n";
if ($isCronJob) {
$details .= "cron: yes\n";
} else {
$details .= "cron: no\n";
}
$details .= "project ID: {$projectId}\n";
if (!$isCronJob) {
$details .= "user: ".$username."\n";
}
$details .= "configuration: {$configName}\n";
$details .= "server: {$serverName}\n";
if ($isCronJob) {
$details .= "cron: yes\n";
} else {
$details .= "cron: no\n";
}

$status = '';
$status = '';

$sql = null;
$record = null;
$event = self::LOG_EVENT;
$sql = null;
$record = null;
$event = self::LOG_EVENT;

$details = "Workflow ETL task submitted \n".$details;
\REDCap::logEvent(self::RUN_LOG_ACTION, $details, $sql, $record, $event, $projectId);

#------------------------------------------------------------------------
# If no server configuration was specified, run on the embedded server
#------------------------------------------------------------------------
#if (empty($serverConfig)) {
# $status = $this->runEmbedded($etlConfig, $isCronJob);
#} else {
# # Run on the specified server
# $status = $serverConfig->run($etlConfig, $isCronJob);
#}
$status = $serverConfig->run($etlConfig, $isCronJob, $this->moduleLog);

$details = "Workflow $workflowName, ETL task # $key, submitted \n".$details;
\REDCap::logEvent(self::RUN_LOG_ACTION, $details, $sql, $record, $event, $projectId);

#------------------------------------------------------------------------
# If no server configuration was specified, run on the embedded server
#------------------------------------------------------------------------
#if (empty($serverConfig)) {
# $status = $this->runEmbedded($etlConfig, $isCronJob);
#} else {
# # Run on the specified server
# $status = $serverConfig->run($etlConfig, $isCronJob);
#}

# calls redcapetl-->run, which calls etl workflow and workflow config
#$status = $serverConfig->run($etlConfig, $isCronJob, $this->moduleLog);
#$workflowProperties['tasks'][$taskName] = $etlConfig;
#$workflowProperties['tasks'][$taskName]['serverConfigName'] = $configName;
#print "=============redcapETLmodule.php, 1646, IN LOOP, etlConfig is ";
#print_r($etlConfig);
#print ".......................................................................";

# does the project_id already exist? $workflowProperties['tasks'][$taskName]['project_id'] = $projectId;
#$status = Workflow($workflowProperties, $isCronJob, $this->moduleLog);

} else {
#skip this task because the user does not have permissions to export

#-----------------------------------------------------
# Set task logging information
#-----------------------------------------------------
$details = '';
$details .= "project ID: {$projectId}\n";
if (!$isCronJob) {
$details .= "user: ".$username."\n";
}
$details .= "server: {$serverName}\n";
if ($isCronJob) {
$details .= "cron: yes\n";
} else {
$details .= "cron: no\n";
}

$sql = null;
$record = null;
$event = self::LOG_EVENT;

$details = "Workflow $workflowName, ETL task # $key, SKIPPED. User does not have export permissions \n".$details;
\REDCap::logEvent(self::RUN_LOG_ACTION, $details, $sql, $record, $event, $projectId);

} #end if (hasPermissionToExport))
} #next task
} #end if (empty(workflowName))

Expand All @@ -1619,6 +1693,9 @@ public function runWorkflow(
\REDCap::logEvent(self::RUN_LOG_ACTION, $details, $sql, $record, $event, $projectId);
}

#print "=============redcapETLmodule.php, 1689, AFTER LOOP, workflowProperties is ";
#print_r($workflowProperties);
#print ".......................................................................";
return $status;
}

Expand Down
31 changes: 16 additions & 15 deletions classes/Configuration.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -721,15 +721,19 @@ public function getGlobalProperties($initialize = false)
unset($properties[self::AUTOGEN_COMBINE_NON_REPEATING_FIELDS]);
unset($properties[self::AUTOGEN_NON_REPEATING_FIELDS_TABLE]);
unset($properties[self::PROJECT_ID]);

unset($properties[self::EMAIL_FROM_ADDRESS]);
unset($properties[self::LOG_FILE]);
unset($properties[self::CA_CERT_FILE]);
unset($properties[self::DB_SSL]);
unset($properties[self::DB_SSL_VERIFY]);
unset($properties[self::PRINT_LOGGING]);


if ($initialize) {
#---------------------------------------
# Remove all default values
#---------------------------------------
$properties[self::CRON_JOB] = null;
$properties[self::DB_LOGGING] = null;
$properties[self::DB_LOG_TABLE] = null;
$properties[self::DB_EVENT_LOG_TABLE] = null;
$properties[self::DB_TYPE] = null;
$properties[self::DB_HOST] = null;
$properties[self::DB_PORT] = null;
Expand All @@ -738,24 +742,21 @@ public function getGlobalProperties($initialize = false)
$properties[self::DB_USERNAME] = null;
$properties[self::DB_PASSWORD] = null;
$properties[self::DB_CONNECTION] = null;
$properties[self::DB_SSL] = null;
$properties[self::DB_SSL_VERIFY] = null;
$properties[self::DB_PRIMARY_KEYS] = null;
$properties[self::DB_FOREIGN_KEYS] = null;
$properties[self::CA_CERT_FILE] = null;
$properties[self::IGNORE_EMPTY_INCOMPLETE_FORMS] = null;
$properties[self::BATCH_SIZE] = null;
$properties[self::IGNORE_EMPTY_INCOMPLETE_FORMS] = null;
$properties[self::TABLE_PREFIX] = null;
$properties[self::LABEL_VIEW_SUFFIX] = null;
$properties[self::POST_PROCESSING_SQL] = null;
$properties[self::PRE_PROCESSING_SQL] = null;
$properties[self::PRINT_LOGGING] = null;
$properties[self::LOG_FILE] = null;
$properties[self::DB_PRIMARY_KEYS] = null;
$properties[self::DB_FOREIGN_KEYS] = null;
$properties[self::DB_LOGGING] = null;
$properties[self::DB_LOG_TABLE] = null;
$properties[self::DB_EVENT_LOG_TABLE] = null;
$properties[self::EMAIL_ERRORS] = null;
$properties[self::EMAIL_SUMMARY] = null;
$properties[self::EMAIL_FROM_ADDRESS] = null;
$properties[self::EMAIL_SUBJECT] = null;
$properties[self::EMAIL_TO_LIST] = null;
$properties[self::POST_PROCESSING_SQL] = null;
$properties[self::PRE_PROCESSING_SQL] = null;
$properties[self::CRON_SERVER] = null;
$properties[self::CRON_SCHEDULE] = null;
}
Expand Down
Loading

0 comments on commit 95496ee

Please sign in to comment.