Skip to content

Commit

Permalink
Merge pull request #147 from lbr38/devel
Browse files Browse the repository at this point in the history 
3.7.11
  • Loading branch information
@lbr38
lbr38 authored Jan 7, 2024
2 parents 9983b49 + d70714b commit 31310f3
Show file tree
Hide file tree
Showing 2 changed files with 121 additions and 127 deletions.
149 changes: 68 additions & 81 deletions www/controllers/Service/Statistic.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,6 @@ public function parseAccessLog()
$size = ftell($file);
}

// usleep(100);
usleep(100000); // 0.1 second
}
}
Expand Down Expand Up @@ -311,23 +310,23 @@ public function processAccessLog()
*/
$queue = $this->statController->getAccessQueue();

/**
* Get all repos
*/
$reposList = $this->repoListingController->list();

/**
* Process access log entries
*/
if (!empty($queue)) {
if (!empty($queue) and !empty($reposList)) {
foreach ($queue as $line) {
$id = $line['Id'];
$type = '';
$dist = '';
$section = '';

/**
* Parse request
*/
$request = explode(' ', $line['Request']);

print_r($request);

/**
* Date and time
*/
Expand Down Expand Up @@ -361,110 +360,98 @@ public function processAccessLog()
$requestGrabber = $request[12];

/**
* Retrieve repo type
*
*
* Case the request matches deb repo pattern
* e.g: GET /repo/ubuntu/jammy/main_preprod/dists/jammy/Release HTTP/1.1
*
* If APT-HTTP or APT-CURL is found in the request grabber, it means that the request is made by apt
* Loop through repos list until the repo called in the request is found
*/
if (preg_match('/APT-HTTP|APT-CURL/', $requestGrabber) and preg_match('#/repo/.*/pool/|/repo/.*/dists/#', $fullRequest)) {
$type = 'deb';
foreach ($reposList as $repo) {
$dist = '';
$section = '';

/**
* Parse repo name, dist, section and env
* Continue if the repo snapshot has no environment, because stats are only generated for snapshots environments
*/
$fullRequestExplode = explode('/', $fullRequest);
$name = $fullRequestExplode[2];
$dist = $fullRequestExplode[3];
$section = explode('_', $fullRequestExplode[4])[0];
$env = explode('_', $fullRequestExplode[4])[1];

/**
* If the repo does not exist, continue
*/
if (!$this->repoController->existsEnv($name, $dist, $section, $env)) {
echo 'Could not find repo for request with Id ' . $id . PHP_EOL;
if (empty($repo['envId'])) {
continue;
}
}

/**
* Case the request matches rpm repo pattern
* e.g: GET /repo/epel_prod/repodata/repomd.xml HTTP/1.1
*
* If APT-HTTP or APT-CURL is NOT found in the request grabber, it means that the request might be made by yum
*/
if (!preg_match('/APT-HTTP|APT-CURL/', $requestGrabber)
and preg_match('/yum.*/', $requestGrabber)
and preg_match('#/repo/.*/packages/|/repo/.*/repodata/#', $fullRequest)) {
$type = 'rpm';

/**
* Parse repo name and env
* Build repository URI path
*/
$fullRequestExplode = explode('/', $fullRequest);
$name = explode('_', $fullRequestExplode[2])[0];
$env = explode('_', $fullRequestExplode[2])[1];

/**
* If the repo does not exist, continue
* Case the repo is a deb repo
*/
if (!$this->repoController->existsEnv($name, '', '', $env)) {
echo 'Could not find repo for request with Id ' . $id . PHP_EOL;
continue;
if ($repo['Package_type'] == 'deb') {
$repoUri = '/repo/' . $repo['Name'] . '/' . $repo['Dist'] . '/' . $repo['Section'] . '_' . $repo['Env'];
}
}

/**
* If the request does not match any of the above patterns, skip it
* It might be a request for a file that is not part of a repository (/gpgkeys/ for example)
*/
if (empty($type)) {
echo 'Could not find repo type for request with Id ' . $id . PHP_EOL;
/**
* Case the repo is a rpm repo
*/
if ($repo['Package_type'] == 'rpm') {
$repoUri = '/repo/' . $repo['Name'] . '_' . $repo['Env'];
}

/**
* Delete the log line from the queue
* Now if the repo URI is found in the request, it means that the request is made for this repo
*/
$this->statController->deleteFromQueue($id);
if (preg_match('#' . $repoUri . '#', $fullRequest)) {
$type = $repo['Package_type'];
$name = $repo['Name'];
$env = $repo['Env'];
if (!empty($repo['Dist']) and !empty($repo['Section'])) {
$dist = $repo['Dist'];
$section = $repo['Section'];
}

// For debugging
// echo 'Date: ' . $date . PHP_EOL;
// echo 'Time: ' . $time . PHP_EOL;
// echo 'Source IP: ' . $sourceIp . PHP_EOL;
// echo 'Source host: ' . $sourceHost . PHP_EOL;
// echo 'Request: ' . $fullRequest . PHP_EOL;
// echo 'Request result: ' . $requestResult . PHP_EOL;
// echo 'Request grabber: ' . $requestGrabber . PHP_EOL;
// echo 'Type: ' . $type . PHP_EOL;
// echo 'Name: ' . $name . PHP_EOL;
// if (!empty($dist) and !empty($section)) {
// echo 'Dist: ' . $dist . PHP_EOL;
// echo 'Section: ' . $section . PHP_EOL;
// }
// echo 'Env: ' . $env . PHP_EOL . PHP_EOL;

continue;
}

// For debugging
echo 'Date: ' . $date . PHP_EOL;
echo 'Time: ' . $time . PHP_EOL;
echo 'Source IP: ' . $sourceIp . PHP_EOL;
echo 'Source host: ' . $sourceHost . PHP_EOL;
echo 'Request: ' . $fullRequest . PHP_EOL;
echo 'Request result: ' . $requestResult . PHP_EOL;
echo 'Request grabber: ' . $requestGrabber . PHP_EOL;
echo 'Type: ' . $type . PHP_EOL;
echo 'Name: ' . $name . PHP_EOL;
if (!empty($dist) and !empty($section)) {
echo 'Dist: ' . $dist . PHP_EOL;
echo 'Section: ' . $section . PHP_EOL;
/**
* Add repo access log to database
*/
if (!empty($date) and !empty($time) and !empty($type) and !empty($name) and isset($dist) and isset($section) and !empty($env) and !empty($sourceHost) and !empty($sourceIp) and !empty($fullRequest) and !empty($requestResult)) {
$this->statController->addAccess($date, $time, $type, $name, $dist, $section, $env, $sourceHost, $sourceIp, $fullRequest, $requestResult);
}

/**
* Delete the log line from the queue now that it has been processed
*/
$this->statController->deleteFromQueue($id);

/**
* Process the next log line
*/
continue 2;
}
}
echo 'Env: ' . $env . PHP_EOL;

/**
* Add repo access log to database
* If the request does not match any of the repo URI patterns, skip it
*/
if (!empty($date) and !empty($time) and !empty($type) and !empty($name) and !empty($env) and !empty($sourceHost) and !empty($sourceIp) and !empty($fullRequest) and !empty($requestResult)) {
$this->statController->addAccess($date, $time, $type, $name, $dist, $section, $env, $sourceHost, $sourceIp, $fullRequest, $requestResult);
}
echo 'Could not find repo for request with Id ' . $id . PHP_EOL;

/**
* Then delete the log line from the queue
* Delete the log line from the queue
*/
$this->statController->deleteFromQueue($id);

unset($id, $line, $dateExplode, $date, $time, $sourceIp, $sourceHost, $fullRequest, $requestResult, $type, $name, $dist, $section, $env);
}
}

sleep(2);
sleep(5);
}
}
}
99 changes: 53 additions & 46 deletions www/update/database/3.7.11.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
*/
$statsDb = new \Models\Connection('stats');
$mystat = new \Controllers\Stat();
$myrepo = new \Controllers\Repo\Repo();
$myrepoListing = new \Controllers\Repo\Listing();

/**
* Quit if 'access' table does not exist
Expand All @@ -22,6 +22,18 @@
return;
}

/**
* Get all repos
*/
$reposList = $myrepoListing->list();

/**
* Quit if no repo
*/
if (empty($reposList)) {
return;
}

/**
* Table migration: splitting access table into two tables (access_deb and access_rpm)
*/
Expand Down Expand Up @@ -58,66 +70,61 @@
$sourceIp = $line['IP'];
$request = str_replace('"', '', $line['Request']);
$requestResult = $line['Request_result'];
$type = '';
$dist = '';
$section = '';

/**
* Try to determine the target repository type, name and environment
* Loop through repos list until the repo called in the request is found
*/
foreach ($reposList as $repo) {
$dist = '';
$section = '';

/**
* Case it's a deb repository
*/
if (preg_match('#/repo/.*/pool/|/repo/.*/dists/|.*\.deb HTTP.*|.*\.dsc HTTP.*|.*\.tar\.gz HTTP.*#', $request)) {
$type = 'deb';
/**
* Continue if the repo snapshot has no environment, because stats are only generated for snapshots environments
*/
if (empty($repo['envId'])) {
continue;
}

/**
* Retrieve name, distribution, section and environment from request
* Build repository URI path
*/
$requestExplode = explode('/', $request);
$name = $requestExplode[2];
$dist = $requestExplode[3];

/**
* Ignore request if section and env are not in position 4
* (this can be the case for some repositories like 'debian-security')
* Case the repo is a deb repo
*/
if (!preg_match('/_/', $requestExplode[4])) {
continue;
if ($repo['Package_type'] == 'deb') {
$repoUri = '/repo/' . $repo['Name'] . '/' . $repo['Dist'] . '/' . $repo['Section'] . '_' . $repo['Env'];
}

$section = explode('_', $requestExplode[4])[0];
$env = explode('_', $requestExplode[4])[1];

/**
* Case it's a rpm repository
*/
} elseif (preg_match('#/repo/.*/packages/|/repo/.*/Packages/|/repo/.*/repodata/|.*\.rpm HTTP.*#', $request)) {
$type = 'rpm';

/**
* Retrieve name and environment from request
* Case the repo is a rpm repo
*/
$requestExplode = explode('/', $request);
$name = explode('_', $requestExplode[2])[0];
$env = explode('_', $requestExplode[2])[1];

/**
* If the request does not match any of the above patterns, skip it
*/
} else {
continue;
}
if ($repo['Package_type'] == 'rpm') {
$repoUri = '/repo/' . $repo['Name'] . '_' . $repo['Env'];
}

/**
* Add line in the new table
*/
if (!empty($date) && !empty($time) && !empty($type) && !empty($name) && !empty($env) && !empty($sourceHost) && !empty($sourceIp) && !empty($request) && !empty($requestResult)) {
$mystat->addAccess($date, $time, $type, $name, $dist, $section, $env, $sourceHost, $sourceIp, $request, $requestResult);
/**
* Now if the repo URI is found in the request, it means that the request is made for this repo
*/
if (!empty($repoUri) and !empty($request)) {
if (preg_match('#' . $repoUri . '#', $request)) {
$type = $repo['Package_type'];
$name = $repo['Name'];
$env = $repo['Env'];
if (!empty($repo['Dist']) and !empty($repo['Section'])) {
$dist = $repo['Dist'];
$section = $repo['Section'];
}

/**
* Add line in the new table
*/
if (!empty($date) && !empty($time) && !empty($type) && !empty($name) && isset($dist) && isset($section) && !empty($env) && !empty($sourceHost) && !empty($sourceIp) && !empty($request) && !empty($requestResult)) {
$mystat->addAccess($date, $time, $type, $name, $dist, $section, $env, $sourceHost, $sourceIp, $request, $requestResult);
}
}
}
}

unset($name, $dist, $section, $env, $sourceHost, $sourceIp, $request, $requestResult, $type, $requestExplode);
}

/**
Expand All @@ -141,6 +148,6 @@
$statsDb->exec("ANALYZE");
$statsDb->close();

unset($statsDb, $mystat, $myrepo);
unset($statsDb, $mystat, $myrepoListing);

echo 'Migration done.' . PHP_EOL;

0 comments on commit 31310f3

Please sign in to comment.