Skip to content

Fixes Error: Workflows with triggers, which have more than 65.535 parameters (or course IDs) with PostgreSQL throw an error (in all versions) #311

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mikeJKU wants to merge 1 commit into
base: develop_500
Choose a base branch
from
Open

Fixes Error: Workflows with triggers, which have more than 65.535 parameters (or course IDs) with PostgreSQL throw an error (in all versions) #311

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 157 additions & 0 deletions classes/local/intersectedRecordset.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
<?php

// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.

/**
* Helper class which intersects multiple moodle record sets.
*
* @package tool_lifecycle
* @copyright 2025 Michael Schink JKU
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace tool_lifecycle\local;

defined('MOODLE_INTERNAL') || die();

class intersectedRecordset implements \Iterator, \Countable {
private $records = [];
private $position = 0;
private $wasFilled = false;

/**
* Constructor: Inits class & intersects passed recordsets.
*
* @param moodle_recordset|array|null $recordsets
* @param string $key
*/
public function __construct($recordsets = null, string $key = 'id') {
if($recordsets !== null) {
if(is_array($recordsets)) {
// For multiple recordsets
foreach($recordsets as $recordset) {
// If recordset is a chunked recordset
if(is_array($recordset)) {
//mtrace('Chunked recordset');
// Create new array for chunked recordset
$chunkedRecords = [];
// For each chunked recordset
foreach($recordset as $chunk_recordset) {
// For each record in chunked recordset
foreach($chunk_recordset as $record) {
if(isset($record->$key)) { $chunkedRecords[$record->$key] = $record; }
}
}
// Add all records of chunked recordsets
$this->add($chunkedRecords, $key);
} else {
//mtrace('Normal recordset');
$this->add($recordset, $key);
}
}
} else { $this->add($recordsets, $key); }
}
}

/**
* Adds recordset & saves intersection of all recordsets.
*
* @param moodle_recordset $recordset
* @param string $key
*/
public function add($recordset, string $key = 'id'): void {
// Add new records to array with key
$newRecords = [];
foreach($recordset as $record) {
if(isset($record->$key)) { $newRecords[$record->$key] = $record; }
}
//mtrace(' Found '.count($newRecords).' records in recordset');
//$recordset->close();

// Store new records without key, if no records were stored & return
if(empty($this->records) && !$this->wasFilled) {
$this->records = array_values($newRecords);
$this->wasFilled = true;

return;
}

// Add existing records to array with key
$existingRecords = [];
foreach($this->records as $record) {
if(isset($record->$key)) { $existingRecords[$record->$key] = $record; }
}

// Intersect existing & new records by keys
$intersectionKeys = array_intersect_key($existingRecords, $newRecords);
// Clear existing records
$this->records = [];
// Store intersected records by keys
foreach($intersectionKeys as $keyValue => $record) {
$this->records[] = $existingRecords[$keyValue];
}
//mtrace('Add - Intersected record sets: '.count($this->records));
}

/**
* Returns current recordset.
*
* @return mixed
*/
public function current(): mixed {
return $this->records[$this->position];
}

/**
* Returns current key (index).
*
* @return int
*/
public function key(): int {
return $this->position;
}

/**
* Moves internal pointer to next recordset.
*/
public function next(): void {
$this->position++;
}

/**
* Returns internal pointer to start.
*/
public function rewind(): void {
$this->position = 0;
}

/**
* Checks if current pointer points to a valid recordset.
*
* @return bool
*/
public function valid(): bool {
return isset($this->records[$this->position]);
}

/**
* Returns the amount of all recordsets.
*
* @return int
*/
public function count(): int {
return count($this->records);
}
}
208 changes: 208 additions & 0 deletions classes/processor.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,213 @@ public function process_course_interactive($processid) {
*/
public function get_course_recordset($triggers, $nositecourse = true, $forcounting = false) {
global $DB, $SESSION;

// Mike
$where = [];
$whereparams = [];
$recordsets = [];
$workflow = false;
foreach ($triggers as $trigger) {
$where_tmp2 = " TRUE ";
$whereparams_tmp2 = [];
if (!$workflow) {
$workflow = workflow_manager::get_workflow($trigger->workflowid);
$andor = ($workflow->andor ?? 0) == 0 ? 'AND' : 'OR';
$where_tmp2 = $andor == 'AND' ? 'true ' : 'false ';
}
$lib = lib_manager::get_automatic_trigger_lib($trigger->subpluginname);
// Exclude specificdate trigger when counting.
if (!($forcounting && $lib->default_response() == trigger_response::triggertime())) {
[$sql, $params] = $lib->get_course_recordset_where($trigger->id);
$sql = preg_replace("/{course}/", "c", $sql, 1);
if (!empty($sql)) {
$where_tmp2 .= " $andor " . $sql;
$whereparams[] = array_merge($whereparams_tmp2, $params);
} else { $whereparams[] = []; }
} else { $whereparams[] = []; }
$where[] = $where_tmp2;
}

$maxparams = 65535;
//$maxparams = 1000;
mtrace('');
mtrace('Start - MAX params: '.$maxparams.', trigger where parts: '.count($where)/*.' '.print_r($where, true)*/.' & params: '.count($whereparams)/*.' '.print_r($whereparams, true)*/);
foreach($whereparams as $key => $whereparam) {
if(count($whereparam) > $maxparams) {
mtrace('More than '.$maxparams.' params with key '.$key.': '.count($whereparam));
// Get where part of params array
$wherepart = $where[$key];
// Get first & last param
$first = ':'.array_key_first($whereparam);
$last = ':'.array_key_last($whereparam);
mtrace(' 1. Get first param '.$first.' & last param '.$last);
// Get where part before first param & after last param (to re-create where part)
$position = strpos($wherepart, $first);
$before = substr($wherepart, 0, $position);
$position = strpos($wherepart, $last);
$after = substr($wherepart, $position + strlen($last));
mtrace(' 2. Re-create where part: '.$before.' <enter-params> '.$after);
// Remove original where part & params
//unset($where[$key]);
//unset($whereparams[$key]);
$where[$key] = [];
$whereparams[$key] = [];
mtrace(' 3. Remove where part & params with key: '.$key);
// Chunk params
$whereparam_chunks = array_chunk($whereparam, $maxparams, true);
mtrace(' 4. Chunk params: '.count($whereparam_chunks)/*.print_r($whereparam_chunks, true))*/.' ('.count($whereparam).'/'.$maxparams.')');
// For each chunk of params
$counter = 0;
foreach($whereparam_chunks as $whereparam_chunk) {
$counter++;
// Create param string of chunk params
$whereparam_chunk_string = implode(',', array_map(function($value) { return ':' . $value; }, array_keys($whereparam_chunk)));
// Re-create where part for chunk
$where_chunk = $before.$whereparam_chunk_string.$after;
if(count($whereparam_chunks) > 10 && $counter == 10 ) { mtrace(' ...'); }
if($counter < 5 || $counter >= (count($whereparam_chunks) - 5)) { mtrace(' 5.'.$counter.' Add chunk query: '.(strlen($where_chunk) > 150 ? substr($where_chunk, 0, 150) . '...' : $where_chunk).' & params: '.count($whereparam_chunk)/*.print_r($whereparam_chunk, true)*/); }
// Add where part & params of chunk
if(count($where) && count($whereparams)) {
$where[$key][] = $where_chunk;
$whereparams[$key][] = $whereparam_chunk;
} else { mtrace('ERROR: Amount of where parts & params are not the same!'); }
}
}
}
mtrace('End - MAX params: '.$maxparams.', trigger where parts: '.count($where)/*.' '.print_r($where, true)*/.' & params '.count($whereparams)/*.' '.print_r($whereparams, true)*/);
mtrace('');
//die();

if ($forcounting) {
foreach ($where as $key => $where_tmp) {
$whereparams_tmp = $whereparams[$key];
// Get course hasprocess and delay with the sql.
$sql = "SELECT c.id,
COALESCE(p.courseid, pe.courseid, 0) as hasprocess,
COALESCE(po.workflowid, peo.workflowid, 0) as hasotherwfprocess,
CASE
WHEN COALESCE(d.delayeduntil, 0) > COALESCE(dw.delayeduntil, 0) THEN d.delayeduntil
WHEN COALESCE(d.delayeduntil, 0) < COALESCE(dw.delayeduntil, 0) THEN dw.delayeduntil
ELSE 0
END as delaycourse
FROM {course} c
LEFT JOIN {tool_lifecycle_process} p ON c.id = p.courseid AND p.workflowid = $workflow->id
LEFT JOIN {tool_lifecycle_proc_error} pe ON c.id = pe.courseid AND pe.workflowid = $workflow->id
LEFT JOIN {tool_lifecycle_process} po ON c.id = po.courseid AND po.workflowid <> $workflow->id
LEFT JOIN {tool_lifecycle_proc_error} peo ON c.id = peo.courseid AND peo.workflowid <> $workflow->id
LEFT JOIN {tool_lifecycle_delayed} d ON c.id = d.courseid
LEFT JOIN {tool_lifecycle_delayed_workf} dw ON
c.id = dw.courseid AND dw.workflowid = $workflow->id
WHERE ";
if(is_array($where_tmp)) {
//mtrace('Chunked recordset: '.count($where_tmp)/*.' '.print_r($where_tmp, true)*/.', params: '.count($whereparams_tmp)/*.' '.print_r($whereparams_tmp, true)*/);
$tmp = [];
foreach($where_tmp as $chunk_key => $chunk_where_tmp) {
// We include delayed courses here anyway, so we only take the site course into account.
if ($nositecourse) {
$chunk_where_tmp = "($chunk_where_tmp) AND c.id <> 1 ";
}
$sql_tmp = $sql.$chunk_where_tmp;
$debugsql = $sql_tmp;
foreach ($whereparams_tmp as $key => $value) {
$debugsql = str_replace(":".$key, $value, $debugsql);
}
$SESSION->debugtriggersql = $debugsql;
$tmp[] = $DB->get_recordset_sql($sql_tmp, $whereparams_tmp[$chunk_key]);
}
$recordsets[] = $tmp;
} else {
//mtrace('Nomrmal recordset: '.$where_tmp.', params: '.count($whereparams_tmp)/*.' '.print_r($whereparams_tmp, true)*/);
// We include delayed courses here anyway, so we only take the site course into account.
if ($nositecourse) {
$where_tmp = "($where_tmp) AND c.id <> 1 ";
}
$sql_tmp = $sql.$where_tmp;
$debugsql = $sql_tmp;
foreach ($whereparams_tmp as $key => $value) {
$debugsql = str_replace(":".$key, $value, $debugsql);
}
$SESSION->debugtriggersql = $debugsql;
$recordsets[] = $DB->get_recordset_sql($sql_tmp, $whereparams_tmp);
}
}

//use tool_lifecycle\local\intersectedRecordset;
$recordsets = new \tool_lifecycle\local\intersectedRecordset($recordsets);
//mtrace('Intersected record sets (for counting): '.count($recordsets));
} else {
foreach ($where as $key => $where_tmp) {
$whereparams_tmp = $whereparams[$key];
// Get only courses which are not part of an existing process.
$sql = "SELECT c.id from {course} c
LEFT JOIN {tool_lifecycle_process} p ON c.id = p.courseid
LEFT JOIN {tool_lifecycle_proc_error} pe ON c.id = pe.courseid
WHERE p.courseid is null AND pe.courseid IS NULL AND ";
if(is_array($where_tmp)) {
//mtrace('Chunked recordset: '.count($where_tmp)/*.' '.print_r($where_tmp, true)*/.', params: '.count($whereparams_tmp)/*.' '.print_r($whereparams_tmp, true)*/);
$tmp = [];
foreach($where_tmp as $chunk_key => $chunk_where_tmp) {
if ($workflow) {
if (!$workflow->includesitecourse) {
$chunk_where_tmp = "($chunk_where_tmp) AND c.id <> 1 ";
}
if (!$workflow->includedelayedcourses) {
$chunk_where_tmp = "($chunk_where_tmp) AND NOT c.id in (select courseid FROM {tool_lifecycle_delayed_workf}
WHERE delayeduntil > :time1 AND workflowid = :workflowid)
AND NOT c.id in (select courseid FROM {tool_lifecycle_delayed} WHERE delayeduntil > :time2) ";
$inparams = ['time1' => time(), 'time2' => time(), 'workflowid' => $workflow->id];
$whereparams_tmp = array_merge($whereparams_tmp, $inparams);
}
}
$sql_tmp = $sql.$chunk_where_tmp;
$debugsql = $sql_tmp;
foreach ($whereparams_tmp as $key => $value) {
$debugsql = str_replace(":".$key, $value, $debugsql);
}
$SESSION->debugtriggersql = $debugsql;
$tmp[] = $DB->get_recordset_sql($sql_tmp, $whereparams_tmp[$chunk_key]);
}
$recordsets[] = $tmp;
} else {
//mtrace('Nomrmal recordset: '.$where_tmp.', params: '.count($whereparams_tmp)/*.' '.print_r($whereparams_tmp, true)*/);
if ($workflow) {
if (!$workflow->includesitecourse) {
$where_tmp = "($where_tmp) AND c.id <> 1 ";
}
if (!$workflow->includedelayedcourses) {
$where_tmp = "($where_tmp) AND NOT c.id in (select courseid FROM {tool_lifecycle_delayed_workf}
WHERE delayeduntil > :time1 AND workflowid = :workflowid)
AND NOT c.id in (select courseid FROM {tool_lifecycle_delayed} WHERE delayeduntil > :time2) ";
$inparams = ['time1' => time(), 'time2' => time(), 'workflowid' => $workflow->id];
$whereparams_tmp = array_merge($whereparams_tmp, $inparams);
}
}
$sql_tmp = $sql.$where_tmp;
$debugsql = $sql_tmp;
foreach ($whereparams_tmp as $key => $value) {
$debugsql = str_replace(":".$key, $value, $debugsql);
}
$SESSION->debugtriggersql = $debugsql;
$recordsets[] = $DB->get_recordset_sql($sql_tmp, $whereparams_tmp);
}
}

//use tool_lifecycle\local\intersectedRecordset;
$recordsets = new \tool_lifecycle\local\intersectedRecordset($recordsets);
//mtrace('Intersected record sets: '.count($recordsets));
}

mtrace('');
mtrace('FINAL recordsets: '.$recordsets->count()/*.' '.print_r($recordsets, true)*/);
mtrace('');
//die();

return $recordsets;
}

/*
public function get_course_recordset($triggers, $nositecourse = true, $forcounting = false) {
global $DB, $SESSION;

$where = " TRUE ";
$whereparams = [];
Expand Down Expand Up @@ -388,6 +595,7 @@ public function get_course_recordset($triggers, $nositecourse = true, $forcounti

return $DB->get_recordset_sql($sql, $whereparams);
}
*/

/**
* Returns the number of courses for a trigger for counting.
Expand Down
Loading