Skip to content

Fixes Error: Workflows with triggers, which have more than 65.535 parameters (or course IDs) with PostgreSQL throw an error (in all versions) #310

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mikeJKU wants to merge 4 commits into
base: develop_405
Choose a base branch
from
Open

Fixes Error: Workflows with triggers, which have more than 65.535 parameters (or course IDs) with PostgreSQL throw an error (in all versions) #310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
25fe67f
ELEARN-1070
mikeJKU Jul 10, 2025
5039a78
ELEARN-1070
mikeJKU Jul 11, 2025
a48a408
ELEARN-1070
mikeJKU Sep 9, 2025
eb61fb5
ELEARN-1070
mikeJKU Sep 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 157 additions & 0 deletions classes/local/intersectedRecordset.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
<?php

// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.

/**
* Helper class which intersects multiple moodle record sets.
*
* @package tool_lifecycle
* @copyright 2025 Michael Schink JKU
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace tool_lifecycle\local;

defined('MOODLE_INTERNAL') || die();

class intersectedRecordset implements \Iterator, \Countable {
private $records = [];
private $position = 0;
private $wasFilled = false;

/**
* Constructor: Inits class & intersects passed recordsets.
*
* @param moodle_recordset|array|null $recordsets
* @param string $key
*/
public function __construct($recordsets = null, string $key = 'id') {
if($recordsets !== null) {
if(is_array($recordsets)) {
// For multiple recordsets
foreach($recordsets as $recordset) {
// If recordset is a chunked recordset
if(is_array($recordset)) {
mtrace('Chunked recordset');
// Create new array for chunked recordset
$chunkedRecords = [];
// For each chunked recordset
foreach($recordset as $chunk_recordset) {
// For each record in chunked recordset
foreach($chunk_recordset as $record) {
if(isset($record->$key)) { $chunkedRecords[$record->$key] = $record; }
}
}
// Add all records of chunked recordsets
$this->add($chunkedRecords, $key);
} else {
mtrace('Normal recordset');
$this->add($recordset, $key);
}
}
} else { $this->add($recordsets, $key); }
}
}

/**
* Adds recordset & saves intersection of all recordsets.
*
* @param moodle_recordset $recordset
* @param string $key
*/
public function add($recordset, string $key = 'id'): void {
// Add new records to array with key
$newRecords = [];
foreach($recordset as $record) {
if(isset($record->$key)) { $newRecords[$record->$key] = $record; }
}
mtrace(' Found '.count($newRecords).' records in recordset');
//$recordset->close();

// Store new records without key, if no records were stored & return
if(empty($this->records) && !$this->wasFilled) {
$this->records = array_values($newRecords);
$this->wasFilled = true;

return;
}

// Add existing records to array with key
$existingRecords = [];
foreach($this->records as $record) {
if(isset($record->$key)) { $existingRecords[$record->$key] = $record; }
}

// Intersect existing & new records by keys
$intersectionKeys = array_intersect_key($existingRecords, $newRecords);
// Clear existing records
$this->records = [];
// Store intersected records by keys
foreach($intersectionKeys as $keyValue => $record) {
$this->records[] = $existingRecords[$keyValue];
}
//mtrace('Add - Intersected record sets: '.count($this->records));
}

/**
* Returns current recordset.
*
* @return mixed
*/
public function current(): mixed {
return $this->records[$this->position];
}

/**
* Returns current key (index).
*
* @return int
*/
public function key(): int {
return $this->position;
}

/**
* Moves internal pointer to next recordset.
*/
public function next(): void {
$this->position++;
}

/**
* Returns internal pointer to start.
*/
public function rewind(): void {
$this->position = 0;
}

/**
* Checks if current pointer points to a valid recordset.
*
* @return bool
*/
public function valid(): bool {
return isset($this->records[$this->position]);
}

/**
* Returns the amount of all recordsets.
*
* @return int
*/
public function count(): int {
return count($this->records);
}
}
134 changes: 118 additions & 16 deletions classes/processor.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,26 +195,80 @@ public function process_course_interactive($processid) {
public function get_course_recordset($triggers, $exclude, $forcounting = false) {
global $DB;

$where = 'true';
// Mike
$where = [];
$whereparams = [];
$recordsets = [];
foreach ($triggers as $trigger) {
$lib = lib_manager::get_automatic_trigger_lib($trigger->subpluginname);
[$sql, $params] = $lib->get_course_recordset_where($trigger->id);
if (!empty($sql)) {
$where .= ' AND ' . $sql;
$whereparams = array_merge($whereparams, $params);
$where[] = 'true AND ' . $sql;
$whereparams[] = $params;
}
}

if (!empty($exclude)) {
[$insql, $inparams] = $DB->get_in_or_equal($exclude, SQL_PARAMS_NAMED);
$where .= " AND NOT {course}.id {$insql}";
$whereparams = array_merge($whereparams, $inparams);
$where[] = "true AND NOT {course}.id {$insql}";
$whereparams[] = $inparams;
}

$maxparams = 65535;
//$maxparams = 1000;
mtrace('');
mtrace('Start - MAX params: '.$maxparams.', trigger where parts: '.count($where)/*.' '.print_r($where, true)*/.' & params: '.count($whereparams)/*.' '.print_r($whereparams, true)*/);
foreach($whereparams as $key => $whereparam) {
if(count($whereparam) > $maxparams) {
mtrace('More than '.$maxparams.' params with key '.$key.': '.count($whereparam));
// Get where part of params array
$wherepart = $where[$key];
// Get first & last param
$first = ':'.array_key_first($whereparam);
$last = ':'.array_key_last($whereparam);
mtrace(' 1. Get first param '.$first.' & last param '.$last);
// Get where part before first param & after last param (to re-create where part)
$position = strpos($wherepart, $first);
$before = substr($wherepart, 0, $position);
$position = strpos($wherepart, $last);
$after = substr($wherepart, $position + strlen($last));
mtrace(' 2. Re-create where part: '.$before.' <enter-params> '.$after);
// Remove original where part & params
//unset($where[$key]);
//unset($whereparams[$key]);
$where[$key] = [];
$whereparams[$key] = [];
mtrace(' 3. Remove where part & params with key: '.$key);
// Chunk params
$whereparam_chunks = array_chunk($whereparam, $maxparams, true);
mtrace(' 4. Chunk params: '.count($whereparam_chunks)/*.print_r($whereparam_chunks, true))*/.' ('.count($whereparam).'/'.$maxparams.')');
// For each chunk of params
$counter = 0;
foreach($whereparam_chunks as $whereparam_chunk) {
$counter++;
// Create param string of chunk params
$whereparam_chunk_string = implode(',', array_map(function($value) { return ':' . $value; }, array_keys($whereparam_chunk)));
// Re-create where part for chunk
$where_chunk = $before.$whereparam_chunk_string.$after;
if(count($whereparam_chunks) > 10 && $counter == 10 ) { mtrace(' ...'); }
if($counter < 5 || $counter >= (count($whereparam_chunks) - 5)) { mtrace(' 5.'.$counter.' Add chunk query: '.(strlen($where_chunk) > 150 ? substr($where_chunk, 0, 150) . '...' : $where_chunk).' & params: '.count($whereparam_chunk)/*.print_r($whereparam_chunk, true)*/); }
// Add where part & params of chunk
if(count($where) && count($whereparams)) {
$where[$key][] = $where_chunk;
$whereparams[$key][] = $whereparam_chunk;
} else { mtrace('ERROR: Amount of where parts & params are not the same!'); }
}
}
}
mtrace('End - MAX params: '.$maxparams.', trigger where parts: '.count($where)/*.' '.print_r($where, true)*/.' & params '.count($whereparams)/*.' '.print_r($whereparams, true)*/);
mtrace('');
//die();

if ($forcounting) {
// Get course hasotherprocess and delay with the sql.
$sql = "SELECT {course}.id,
foreach ($where as $key => $where_tmp) {
$whereparams_tmp = $whereparams[$key];
// Get course hasotherprocess and delay with the sql.
$sql = "SELECT {course}.id,
COALESCE(p.courseid, pe.courseid, 0) as hasprocess,
CASE
WHEN COALESCE(p.workflowid, 0) > COALESCE(pe.workflowid, 0) THEN p.workflowid
Expand All @@ -232,17 +286,63 @@ public function get_course_recordset($triggers, $exclude, $forcounting = false)
LEFT JOIN {tool_lifecycle_proc_error} pe ON {course}.id = pe.courseid
LEFT JOIN {tool_lifecycle_delayed} d ON {course}.id = d.courseid
LEFT JOIN {tool_lifecycle_delayed_workf} dw ON {course}.id = dw.courseid
WHERE " . $where;
WHERE ";

if(is_array($where_tmp)) {
//mtrace('Chunked recordset: '.count($where_tmp)/*.' '.print_r($where_tmp, true)*/.', params: '.count($whereparams_tmp)/*.' '.print_r($whereparams_tmp, true)*/);
$tmp = [];
foreach($where_tmp as $chunk_key => $chunk_where_tmp) {
$sql_tmp = $sql.$chunk_where_tmp;
$tmp[] = $DB->get_recordset_sql($sql_tmp, $whereparams_tmp[$chunk_key]);
}
$recordsets[] = $tmp;
} else {
//mtrace('Nomrmal recordset: '.$where_tmp.', params: '.count($whereparams_tmp)/*.' '.print_r($whereparams_tmp, true)*/);
$sql_tmp = $sql.$where_tmp;
$recordsets[] = $DB->get_recordset_sql($sql_tmp, $whereparams_tmp);
}
}

//use tool_lifecycle\local\intersectedRecordset;
$recordsets = new \tool_lifecycle\local\intersectedRecordset($recordsets);
//mtrace('Intersected record sets (for counting): '.count($recordsets));
} else {
// Get only courses which are not part of an existing process.
$sql = 'SELECT {course}.id from {course} '.
'LEFT JOIN {tool_lifecycle_process} '.
'ON {course}.id = {tool_lifecycle_process}.courseid '.
'LEFT JOIN {tool_lifecycle_proc_error} pe ON {course}.id = pe.courseid ' .
'WHERE {tool_lifecycle_process}.courseid is null AND ' .
'pe.courseid IS NULL AND '. $where;
foreach ($where as $key => $where_tmp) {
$whereparams_tmp = $whereparams[$key];
// Get only courses which are not part of an existing process.
$sql = 'SELECT {course}.id from {course} '.
'LEFT JOIN {tool_lifecycle_process} '.
'ON {course}.id = {tool_lifecycle_process}.courseid '.
'LEFT JOIN {tool_lifecycle_proc_error} pe ON {course}.id = pe.courseid ' .
'WHERE {tool_lifecycle_process}.courseid is null AND ' .
'pe.courseid IS NULL AND ';

if(is_array($where_tmp)) {
//mtrace('Chunked recordset: '.count($where_tmp)/*.' '.print_r($where_tmp, true)*/.', params: '.count($whereparams_tmp)/*.' '.print_r($whereparams_tmp, true)*/);
$tmp = [];
foreach($where_tmp as $chunk_key => $chunk_where_tmp) {
$sql_tmp = $sql.$chunk_where_tmp;
$tmp[] = $DB->get_recordset_sql($sql_tmp, $whereparams_tmp[$chunk_key]);
}
$recordsets[] = $tmp;
} else {
//mtrace('Nomrmal recordset: '.$where_tmp.', params: '.count($whereparams_tmp)/*.' '.print_r($whereparams_tmp, true)*/);
$sql_tmp = $sql.$where_tmp;
$recordsets[] = $DB->get_recordset_sql($sql_tmp, $whereparams_tmp);
}
}

//use tool_lifecycle\local\intersectedRecordset;
$recordsets = new \tool_lifecycle\local\intersectedRecordset($recordsets);
//mtrace('Intersected record sets: '.count($recordsets));
}
return $DB->get_recordset_sql($sql, $whereparams);

mtrace('');
mtrace('FINAL recordsets: '.$recordsets->count()/*.' '.print_r($recordsets, true)*/);
mtrace('');
//die();

return $recordsets;
}

/**
Expand Down Expand Up @@ -435,6 +535,8 @@ public function get_count_of_courses_to_trigger_for_workflow($workflow) {
$all->delayedcourses = $delayedcourses; // Delayed courses for workflow and globally. Excluded per default.
$all->used = $usedcourses;
$all->nextrun = $nextrun;
// Mike
mtrace('Courses triggered by workflow: '.count($all->triggered));
$amounts['all'] = $all;
return $amounts;
}
Expand Down