Skip to content

Commit

Permalink
Refactor progress into shared search class
Browse files Browse the repository at this point in the history
  • Loading branch information
@bwalkerl @petersistrom
bwalkerl authored and petersistrom committed Nov 15, 2024
1 parent 1fab430 commit 65fe42c
Show file tree
Hide file tree
Showing 4 changed files with 150 additions and 170 deletions.
88 changes: 0 additions & 88 deletions classes/db_search.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,6 @@ class db_search extends search {
/** @var array excludetables from config. */
protected $excludetables = null;

/**
* @var \stdClass tracking of current status */
protected $status = null;

/**
* Return the definition of the properties of this model.
*
Expand Down Expand Up @@ -173,88 +169,4 @@ public function get_min_search_length(): int {
$minsearch = empty($this->get('regex')) ? $this->get('search') : $this->get('prematch');
return strlen($minsearch);
}

/**
* Updates a progress bar using the current status.
* @param string $table table being searched
* @param string $colname column being searched
* @return void
*/
public function update_progress_bar(string $table, string $colname): void {
if (isset($this->status) && isset($this->status->progressbar)) {
$message = "Searching in $table:$colname";
$this->status->progressbar->update($this->status->rowcount, $this->status->totalrows, $message);
}
}

/**
* Updates the tracking status of a search.
* @param int $rowcount number of rows that have been searched
* @param int $matches matches found
* @throws \coding_exception
* @return void
*/
public function update_status(int $rowcount, int $matches): void {
if (!isset($this->status)) {
throw new \coding_exception('Status has not been initalised');
}

// Update row count.
$this->status->rowcount = $rowcount;

// Only save update search progress every 10 seconds or 5 percent.
$time = time();
$percent = round(100 * $rowcount / $this->status->totalrows, 2);
if ($time > $this->status->prevtime + 10 || $percent > $this->status->prevpercent + 5) {
$this->set('progress', $percent);
$this->set('matches', $matches);
$this->save();
$this->status->prevtime = $time;
$this->status->prevpercent = $percent;
}
}

/**
* Marks a search as having started and initialises tracking.
* @param int $totalrows estimate of total rows being searched
* @return void
*/
public function mark_started(int $totalrows): void {
$this->set('timestart', time());
$this->save();

// Setup tracking.
$status = new \stdClass();
$status->prevtime = time();
$status->prevpercent = 0;
$status->rowcount = 0;
$status->totalrows = $totalrows;
$status->progressbar = null;

// If called from CLI, add a progress bar.
if ($this->get('origin') === 'cli') {
$status->progressbar = new \progress_bar();
$status->progressbar->create();
}
$this->status = $status;
}

/**
* Saves the final values and marks a search as finished.
*
* @param int $matches matches found
* @param string $output
* @return void
*/
public function mark_finished(int $matches, string $output = ''): void {
// Update progress bar.
if (isset($this->status) && isset($this->status->progress)) {
$this->status->progress->update_full(100, "Finished saving searches into $output");
}

$this->set('timeend', time());
$this->set('progress', 100);
$this->set('matches', $matches);
$this->save();
}
}
63 changes: 13 additions & 50 deletions classes/file_search.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,12 +161,12 @@ public static function files(files $record, string $output = '',
global $DB;
\core_php_time_limit::raise();
raise_memory_limit(MEMORY_HUGE);
$processing = true;
$criteria = self::get_criteria($record);

$id = $record->get('id');
$logmessage = "Advanced search in files, job $id.";
$shard = $record->is_shard();
$filename = $record->get_filename();
// Create a shared temp output directory.
if (!$output) {
$tempfile = true;
Expand Down Expand Up @@ -213,11 +213,10 @@ public static function files(files $record, string $output = '',

mtrace($logmessage);
$record->set('timestart', time());
$updatetime = time();
$updatepercent = 0;
$filecount = 0;
$total = $DB->get_record_sql("SELECT COUNT('x') total FROM {files} f WHERE " . $whereclause, $params);
$totalfiles = $total->total;
$record->mark_started($totalfiles);
$sql = "
SELECT
f.id, f.component, f.filearea, f.contextid, f.itemid, f.filename, f.filepath, f.mimetype,
Expand All @@ -234,52 +233,28 @@ public static function files(files $record, string $output = '',
";
$fileset = $DB->get_recordset_sql($sql, $params);
foreach ($fileset as $filerecord) {
$record->update_progress_bar("Searching in $filerecord->component:$filerecord->filename");
$matchcount += self::search_file($filerecord, $criteria, $stream);
$filecount ++;
$time = time();
$percent = round(100 * $filecount / $totalfiles, 2);
if ($time > $updatetime + 10 || $percent > $updatepercent + 5) {
// Update progress bar after 5 percent or 10 seconds.
$record->set('progress', $percent);
$record->set('matches', $matchcount);
if ( ! \tool_advancedreplace\files::record_exists($id) ) {
// If record has gone, exit the job.
break;
}
$record->update();
$updatetime = $time;
$updatepercent = $percent;
// Update status. If this returns false, the record is gone so stop searching.
if (!$processing = $record->update_status($filecount, $matchcount)) {
break;
}

}
$fileset->close();
fclose($stream);

if (\tool_advancedreplace\files::record_exists($id) ) {
$record->set('timeend', time());
$record->set('progress', 100);
$record->set('matches', $matchcount);
$record->update();

// Save as pluginfile.
if (!empty($matchcount) && !$shard) {
$fs = get_file_storage();
$fileinfo = [
'contextid' => \context_system::instance()->id,
'component' => 'tool_advancedreplace',
'filearea' => 'files',
'itemid' => $id,
'filepath' => '/',
'filename' => $filename,
];
$fs->create_file_from_pathname($fileinfo, $output);
}
if ($processing) {
$record->mark_finished($matchcount);
$record->save_pluginfile($output);
}
// Remove temp file.
if (isset($tempfile) && file_exists($output) && !$shard) {
@unlink($output);
}

if ($shard) {
if ($processing && $shard) {
$parent = $record->get_parent();
if (isset($parent) && $parent->shards_finished()) {
self::combine_shard_output($parent);
Expand All @@ -303,10 +278,7 @@ public static function combine_shard_output(files $parent): void {
}

// Update parent.
$parent->set('timeend', time());
$parent->set('matches', $matches);
$parent->set('progress', 100);
$parent->update();
$parent->mark_finished($matches);

if (!empty($matches)) {
// Copy data into one csv.
Expand Down Expand Up @@ -335,16 +307,7 @@ public static function combine_shard_output(files $parent): void {
fclose($output);

// Create new pluginfile.
$fs = get_file_storage();
$fileinfo = [
'contextid' => \context_system::instance()->id,
'component' => 'tool_advancedreplace',
'filearea' => 'files',
'itemid' => $parent->get('id'),
'filepath' => '/',
'filename' => $parent->get_filename(),
];
$fs->create_file_from_pathname($fileinfo, $outputpath);
$parent->save_pluginfile($output);
}

// Remove old temp files.
Expand Down
27 changes: 8 additions & 19 deletions classes/helper.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,8 @@ private static function get_column_info(string $table, string $columnname): ?dat
* @return void
*/
public static function search_db(db_search $search, string $output = ''): void {
$processing = true;

// Create a shared temp output directory.
if (!$output) {
$tempfile = true;
Expand All @@ -449,7 +451,6 @@ public static function search_db(db_search $search, string $output = ''): void {

// Start output.
$fp = fopen($output, 'w');
$searchid = $search->get('id');
// Show header.
if (!$search->get('summary')) {
fputcsv($fp, ['table', 'column', 'courseid', 'shortname', 'id', 'match', 'replace', 'link']);
Expand All @@ -474,7 +475,7 @@ public static function search_db(db_search $search, string $output = ''): void {
$colstart = time();

// Show the table and column being searched.
$search->update_progress_bar($table, $colname);
$search->update_progress_bar("Searching in $table:$colname");

// Perform the search.
$results = self::search_column($search, $table, $column, $fp);
Expand All @@ -496,11 +497,10 @@ public static function search_db(db_search $search, string $output = ''): void {
];
}

if ( ! \tool_advancedreplace\db_search::record_exists($searchid) ) {
// The control row has been deleted, so we should exit.
// Update status. If this returns false, the record is gone so stop searching.
if (!$processing = $search->update_status($rowcount, $matches)) {
break 2;
}
$search->update_status($rowcount, $matches);
}
}

Expand All @@ -514,21 +514,10 @@ public static function search_db(db_search $search, string $output = ''): void {
mtrace(sprintf($format, $log->table, $log->column, $log->rows, $log->matches, $log->time));
}
}
if (\tool_advancedreplace\db_search::record_exists($searchid) ) {

if ($processing) {
$search->mark_finished($matches, $output);
// Save as pluginfile.
if (!empty($matches)) {
$fs = get_file_storage();
$fileinfo = [
'contextid' => \context_system::instance()->id,
'component' => 'tool_advancedreplace',
'filearea' => 'search',
'itemid' => $search->get('id'),
'filepath' => '/',
'filename' => $search->get_filename(),
];
$fs->create_file_from_pathname($fileinfo, $output);
}
$search->save_pluginfile($output);
}
// Remove temp file.
if (isset($tempfile) && file_exists($output)) {
Expand Down
Loading

0 comments on commit 65fe42c

Please sign in to comment.