Implement data retention policy (#11)

Co-authored-by: Marko Ivančić <marko.ivancic@srce.hr>

bin/test.php

 #!/usr/bin/env php
 <?php
-// TODO mivanci remove this file
+// TODO mivanci remove this file before release
 declare(strict_types=1);
 
 use SimpleSAML\Module\accounting\Entities\Authentication\Event;
@@ -34,7 +34,7 @@ echo $newLine;
 
 $spinnerChars = ['|', '/', '-', '\\'];
 
-/**
+/**/
 echo 'Starting simulating MySQL: ';
 $mysqlStartTime = new DateTime();
 echo $mysqlStartTime->format(DateTime::ATOM);
@@ -87,7 +87,7 @@ for ($i = 1; $i <= $numberOfItems; $i++) {
 echo $newLine;
 echo $newLine;
 
-*/
+
 echo 'Starting simulating Redis: ';
 $redisStartTime = new DateTime();
 echo $redisStartTime->format(DateTime::ATOM);

config-templates/module_accounting.php

@@ -40,14 +40,6 @@ $config = [
          */
         //ModuleConfiguration\AccountingProcessingType::VALUE_ASYNCHRONOUS,
 
-    /**
-     * Cron tags.
-     *
-     * Job runner tag designates the cron tag to use when running accounting jobs. Make sure to add this tag to
-     * the cron module configuration in case of the 'asynchronous' accounting processing type.
-     */
-    ModuleConfiguration::OPTION_CRON_TAG_FOR_JOB_RUNNER => 'accounting_job_runner',
-
     /**
      * Jobs store class. In case of the 'asynchronous' accounting processing type, this determines which class
      * will be used to store jobs. The class must implement Stores\Interfaces\JobsStoreInterface.
@@ -79,14 +71,12 @@ $config = [
          */
         Trackers\Authentication\DoctrineDbal\Versioned\Tracker::class,
 
-
     /**
      * Additional trackers to run besides default data tracker. These trackers will typically only process and
      * persist authentication data to proper data store, and won't be used to display data in SSP UI.
      * These tracker classes must implement Trackers\Interfaces\AuthenticationDataTrackerInterface.
      */
     ModuleConfiguration::OPTION_ADDITIONAL_TRACKERS => [
-        // TODO mivanci at least one more tracker and its connection
         // tracker-class
     ],
 
@@ -164,17 +154,13 @@ $config = [
 
     /**
      * Job runner fine-grained configuration options.
-     */
-
-    /**
-     * Maximum execution time for the job runner.
      *
-     * You can use this option to limit job runner activity by combining when the job runner will run (using
-     * cron configuration) and how long the job runner will be active (execution time). This can be null,
-     * meaning it will run indefinitely, or can be set as a duration for DateInterval, examples being
-     * below. Note that when the job runner is run using Cron user interface in SimpleSAMLphp, the
-     * duration will be taken from the 'max_execution_time' ini setting, and will override this
-     * setting if ini setting is shorter.
+     * Maximum execution time for the job runner. You can use this option to limit job runner activity by combining
+     * when the job runner will run (using cron configuration) and how long the job runner will be active
+     * (execution time). This can be null, meaning it will run indefinitely, or can be set as a duration
+     * for DateInterval, examples being below. Note that when the job runner is run using Cron user
+     * interface in SimpleSAMLphp, the duration will be taken from the 'max_execution_time' ini
+     * setting, and will override this setting if ini setting is shorter.
      * @see https://www.php.net/manual/en/dateinterval.construct.php
      */
     ModuleConfiguration::OPTION_JOB_RUNNER_MAXIMUM_EXECUTION_TIME => null,
@@ -189,4 +175,32 @@ $config = [
      * backend store. If the value is null, there will be no pause.
      */
     ModuleConfiguration::OPTION_JOB_RUNNER_SHOULD_PAUSE_AFTER_NUMBER_OF_JOBS_PROCESSED => 10,
+
+    /**
+     * Tracker data retention policy.
+     *
+     * Determines how long the tracked data will be stored. If null, data will be stored indefinitely. Otherwise, it
+     * can be set as a duration for DateInterval, examples being below. For this to work, a cron tag must also
+     * be configured.
+     */
+    ModuleConfiguration::OPTION_TRACKER_DATA_RETENTION_POLICY => null,
+    //ModuleConfiguration::OPTION_TRACKER_DATA_RETENTION_POLICY => 'P30D', // 30 days
+    //ModuleConfiguration::OPTION_TRACKER_DATA_RETENTION_POLICY => 'P6M', // 6 months
+    //ModuleConfiguration::OPTION_TRACKER_DATA_RETENTION_POLICY => 'P1Y', // 1 year
+
+
+    /**
+     * Cron tags.
+     *
+     * Job runner tag designates the cron tag to use when running accounting jobs. Make sure to add this tag to
+     * the cron module configuration in case of the 'asynchronous' accounting processing type.
+     */
+    ModuleConfiguration::OPTION_CRON_TAG_FOR_JOB_RUNNER => 'accounting_job_runner',
+
+    /**
+     * Tracker data retention policy tag designates the cron tag to use for enforcing data retention policy. Make sure
+     * to add this tag to the cron module configuration if data retention policy is different to null.
+     */
+    ModuleConfiguration::OPTION_CRON_TAG_FOR_TRACKER_DATA_RETENTION_POLICY =>
+        'accounting_tracker_data_retention_policy',
 ];

hooks/hook_adminmenu.php

@@ -19,6 +19,10 @@ function accounting_hook_adminmenu(\SimpleSAML\XHTML\Template &$template): void
         ],
     ];
 
+    if (!isset($template->data[$menuKey]) || !is_array($template->data[$menuKey])) {
+        return;
+    }
+
     // Use array_splice to put our entry before the "Log out" entry.
     array_splice($template->data[$menuKey], -1, 0, $profilePageEntry);
 

hooks/hook_configpage.php

@@ -11,7 +11,13 @@ function accounting_hook_configpage(Template &$template): void
 {
     $moduleRoutesHelper = new ModuleRoutesHelper();
 
-    $template->data['links'][] = [
+    $dataLinksKey = 'links';
+
+    if (!isset($template->data[$dataLinksKey]) || !is_array($template->data[$dataLinksKey])) {
+        return;
+    }
+
+    $template->data[$dataLinksKey][] = [
         'href' => $moduleRoutesHelper->getUrl(ModuleRoutesHelper::PATH_ADMIN_CONFIGURATION_STATUS),
         'text' => Translate::noop('Profile Page configuration status'),
     ];

hooks/hook_cron.php

@@ -2,20 +2,33 @@
 
 declare(strict_types=1);
 
+use Psr\Log\LoggerInterface;
+use SimpleSAML\Configuration;
+use SimpleSAML\Module\accounting\Services\HelpersManager;
 use SimpleSAML\Module\accounting\Services\JobRunner;
 use SimpleSAML\Module\accounting\ModuleConfiguration;
+use SimpleSAML\Module\accounting\Services\Logger;
+use SimpleSAML\Module\accounting\Trackers\Builders\AuthenticationDataTrackerBuilder;
 
 function accounting_hook_cron(array &$cronInfo): void
 {
     $moduleConfiguration = new ModuleConfiguration();
+    $logger = new Logger();
 
+    /** @var ?string $currentCronTag */
     $currentCronTag = $cronInfo['tag'] ?? null;
 
-    $cronTagForJobRunner = $moduleConfiguration->getCronTagForJobRunner();
+    if (!isset($cronInfo['summary']) || !is_array($cronInfo['summary'])) {
+        $cronInfo['summary'] = [];
+    }
 
+    /**
+     * Job runner handling.
+     */
+    $cronTagForJobRunner = $moduleConfiguration->getCronTagForJobRunner();
     try {
         if ($currentCronTag === $cronTagForJobRunner) {
-            $state = (new JobRunner($moduleConfiguration, \SimpleSAML\Configuration::getConfig()))->run();
+            $state = (new JobRunner($moduleConfiguration, Configuration::getConfig()))->run();
             foreach ($state->getStatusMessages() as $statusMessage) {
                 $cronInfo['summary'][] = $statusMessage;
             }
@@ -31,4 +44,48 @@ function accounting_hook_cron(array &$cronInfo): void
         $message = 'Job runner error: ' . $exception->getMessage();
         $cronInfo['summary'][] = $message;
     }
+
+    if (!isset($cronInfo['summary']) || !is_array($cronInfo['summary'])) {
+        $cronInfo['summary'] = [];
+    }
+
+    /**
+     * Tracker data retention policy handling.
+     */
+    $cronTagForTrackerDataRetentionPolicy = $moduleConfiguration->getCronTagForTrackerDataRetentionPolicy();
+    try {
+        if (
+            $currentCronTag === $cronTagForTrackerDataRetentionPolicy &&
+            ($retentionPolicy = $moduleConfiguration->getTrackerDataRetentionPolicy()) !== null
+        ) {
+            $helpersManager = new HelpersManager();
+            $message = sprintf('Handling data retention policy.');
+            $logger->info($message);
+            $cronInfo['summary'][] = $message;
+            handleDataRetentionPolicy($moduleConfiguration, $logger, $helpersManager, $retentionPolicy);
+        }
+    } catch (Throwable $exception) {
+        $message = 'Error enforcing tracker data retention policy: ' . $exception->getMessage();
+        $cronInfo['summary'][] = $message;
+    }
+}
+
+function handleDataRetentionPolicy(
+    ModuleConfiguration $moduleConfiguration,
+    LoggerInterface $logger,
+    HelpersManager $helpersManager,
+    DateInterval $retentionPolicy
+): void {
+    // Handle default data tracker and provider
+    (new AuthenticationDataTrackerBuilder($moduleConfiguration, $logger, $helpersManager))
+        ->build($moduleConfiguration->getDefaultDataTrackerAndProviderClass())
+        ->enforceDataRetentionPolicy($retentionPolicy);
+
+    $additionalTrackers = $moduleConfiguration->getAdditionalTrackers();
+
+    foreach ($additionalTrackers as $tracker) {
+        (new AuthenticationDataTrackerBuilder($moduleConfiguration, $logger, $helpersManager))
+            ->build($tracker)
+            ->enforceDataRetentionPolicy($retentionPolicy);
+    }
 }
\ No newline at end of file

psalm.xml

@@ -11,6 +11,7 @@
         <directory name="config-templates" />
         <directory name="tests" />
         <directory name="www" />
+        <directory name="hooks" />
 
         <ignoreFiles>
             <directory name="vendor" />

routing/routes/routes.yml

 
-accounting-setup:
-    path:       /setup
-    defaults:   { _controller: 'SimpleSAML\Module\accounting\Http\Controllers\Test::setup' }
-
-# TODO mivanci delete
-accounting-job-runner:
-    path:       /job-runner
-    defaults:   { _controller: 'SimpleSAML\Module\accounting\Http\Controllers\Test::jobRunner' }
+# TODO mivanci delete test route
+accounting-test:
+    path:       /test
+    defaults:   { _controller: 'SimpleSAML\Module\accounting\Http\Controllers\Test::test' }
 
 accounting-admin-configuration-status:
     path:       /admin/configuration/status

src/Auth/Process/Accounting.php

@@ -72,7 +72,6 @@ class Accounting extends ProcessingFilter
                 $this->moduleConfiguration->getAdditionalTrackers()
             );
 
-            /** @var string $tracker */
             foreach ($configuredTrackers as $tracker) {
                 ($this->authenticationDataTrackerBuilder->build($tracker))->process($authenticationEvent);
             }

src/Helpers/RandomHelper.php

 <?php
 
+declare(strict_types=1);
+
 namespace SimpleSAML\Module\accounting\Helpers;
 
 class RandomHelper
@@ -8,8 +10,10 @@ class RandomHelper
     {
         try {
             return random_int($minimum, $maximum);
+            // @codeCoverageIgnoreStart
         } catch (\Throwable $exception) {
             return mt_rand($minimum, $maximum);
+            // @codeCoverageIgnoreEnd
         }
     }
 }

src/Http/Controllers/Admin/Configuration.php

@@ -55,6 +55,7 @@ class Configuration
         $configurationValidationErrors = null;
         $jobsStore = null;
         $defaultDataTrackerAndProvider = null;
+        $additionalTrackers = [];
         $setupNeeded = false;
         $runSetup = $request->query->has('runSetup');
 
@@ -87,7 +88,21 @@ class Configuration
                     }
                 }
             }
-            // TODO mivanci setup for additional trackers from configuration
+
+            foreach ($moduleConfiguration->getAdditionalTrackers() as $trackerClass) {
+                $additionalTrackerInstance =
+                    (new AuthenticationDataTrackerBuilder($moduleConfiguration, $this->logger, $this->helpersManager))
+                    ->build($trackerClass);
+
+                if ($additionalTrackerInstance->needsSetup()) {
+                    if ($runSetup) {
+                        $additionalTrackerInstance->runSetup();
+                    } else {
+                        $setupNeeded = true;
+                    }
+                }
+                $additionalTrackers[$trackerClass] = $additionalTrackerInstance;
+            }
         } catch (Throwable $exception) {
             $configurationValidationErrors = $exception->getMessage();
         }
@@ -97,6 +112,7 @@ class Configuration
             'configurationValidationErrors' => $configurationValidationErrors,
             'jobsStore' => $jobsStore,
             'defaultDataTrackerAndProvider' => $defaultDataTrackerAndProvider,
+            'additionalTrackers' => $additionalTrackers,
             'setupNeeded' => $setupNeeded,
             'profilePageUri' => $this->helpersManager->getModuleRoutesHelper()
                 ->getUrl(ModuleRoutesHelper::PATH_USER_PERSONAL_DATA),

src/Http/Controllers/Test.php

@@ -12,6 +12,7 @@ use SimpleSAML\Locale\Translate;
 use SimpleSAML\Module\accounting\Entities\Authentication\Event;
 use SimpleSAML\Module\accounting\Entities\Authentication\State;
 use SimpleSAML\Module\accounting\ModuleConfiguration;
+use SimpleSAML\Module\accounting\Services\HelpersManager;
 use SimpleSAML\Module\accounting\Stores\Builders\JobsStoreBuilder;
 use SimpleSAML\Module\accounting\Stores\Data\Authentication\DoctrineDbal\Versioned\Store;
 use SimpleSAML\Module\accounting\Trackers\Authentication\DoctrineDbal\Versioned\Tracker;
@@ -30,6 +31,7 @@ class Test
     protected Session $session;
     protected ModuleConfiguration $moduleConfiguration;
     protected LoggerInterface $logger;
+    protected HelpersManager $helpersManager;
 
     /**
      * @param SspConfiguration $sspConfiguration
@@ -41,12 +43,14 @@ class Test
         SspConfiguration $sspConfiguration,
         Session $session,
         ModuleConfiguration $moduleConfiguration,
-        LoggerInterface $logger
+        LoggerInterface $logger,
+        HelpersManager $helpersManager
     ) {
         $this->sspConfiguration = $sspConfiguration;
         $this->session = $session;
         $this->moduleConfiguration = $moduleConfiguration;
         $this->logger = $logger;
+        $this->helpersManager = $helpersManager;
     }
 
     /**
@@ -54,90 +58,18 @@ class Test
      * @return Template
      * @throws Exception
      */
-    public function setup(Request $request): Template
+    public function test(Request $request): Template
     {
         $template = new Template($this->sspConfiguration, 'accounting:test.twig');
 
-        $jobsStore = (new JobsStoreBuilder($this->moduleConfiguration, $this->logger))
-            ->build($this->moduleConfiguration->getJobsStoreClass());
+        $retentionPolicy = new \DateInterval('P4D');
 
-        // TODO refactor to tracker builder
-        $dataStoreConnectionKey = $this->moduleConfiguration->getClassConnectionKey(Tracker::class);
-        $dataStore = Store::build($this->moduleConfiguration, $this->logger, $dataStoreConnectionKey);
+        (new AuthenticationDataTrackerBuilder($this->moduleConfiguration, $this->logger, $this->helpersManager))
+            ->build($this->moduleConfiguration->getDefaultDataTrackerAndProviderClass())
+            ->enforceDataRetentionPolicy($retentionPolicy);
 
-        $jobsStoreNeedsSetup = $jobsStore->needsSetup();
-        $jobsStoreSetupRan = false;
+        die('end');
 
-        $dataStoreNeedsSetup = $dataStore->needsSetup();
-        $dataStoreSetupRan = false;
-
-
-        if ($jobsStoreNeedsSetup && $request->query->has('setup')) {
-            $this->logger->info('Jobs Store setup ran.');
-            $jobsStore->runSetup();
-            $jobsStoreSetupRan = true;
-        }
-
-        if ($dataStoreNeedsSetup && $request->query->has('setup')) {
-            $this->logger->info('Data Store setup ran.');
-            $dataStore->runSetup();
-            $dataStoreSetupRan = true;
-        }
-
-        $sampleStateArray = ['Responder'=>[0=>'\\SimpleSAML\\Module\\saml\\IdP\\SAML2',1=>'sendResponse',],'\\SimpleSAML\\Auth\\State.exceptionFunc'=>[0=>'\\SimpleSAML\\Module\\saml\\IdP\\SAML2',1=>'handleAuthError',],'\\SimpleSAML\\Auth\\State.restartURL'=>'https://localhost.someone.from.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/saml2/idp/SSOService.php?spentityid=https%3A%2F%2Fpc-example.org.hr%3A9074%2Fsimplesamlphp%2Fsimplesamlphp-2-beta-git%2Fmodule.php%2Fsaml%2Fsp%2Fmetadata.php%2Fdefault-sp&RelayState=https%3A%2F%2Flocalhost.someone.from.hr%3A9074%2Fsimplesamlphp%2Fsimplesamlphp-2-beta-git%2Fmodule.php%2Fadmin%2Ftest%2Fdefault-sp&cookieTime=1660912195','SPMetadata'=>['SingleLogoutService'=>[0=>['Binding'=>'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect','Location'=>'https://pc-example.org.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/module.php/saml/sp/singleLogoutService/default-sp',],],'AssertionConsumerService'=>[0=>['Binding'=>'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST','Location'=>'https://pc-example.org.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/module.php/saml/sp/assertionConsumerService/default-sp','index'=>0,],1=>['Binding'=>'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Artifact','Location'=>'https://pc-example.org.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/module.php/saml/sp/assertionConsumerService/default-sp','index'=>1,],],'contacts'=>[0=>['emailAddress'=>'example@org.hr','givenName'=>'MarkoIvančić','contactType'=>'technical',],],'entityid'=>'https://pc-example.org.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/module.php/saml/sp/metadata.php/default-sp','metadata-index'=>'https://pc-example.org.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/module.php/saml/sp/metadata.php/default-sp','metadata-set'=>'saml20-sp-remote',],'saml:RelayState'=>'https://localhost.someone.from.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/module.php/admin/test/default-sp','saml:RequestId'=>null,'saml:IDPList'=>[],'saml:ProxyCount'=>null,'saml:RequesterID'=>null,'ForceAuthn'=>false,'isPassive'=>false,'saml:ConsumerURL'=>'https://pc-example.org.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/module.php/saml/sp/assertionConsumerService/default-sp','saml:Binding'=>'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST','saml:NameIDFormat'=>null,'saml:AllowCreate'=>true,'saml:Extensions'=>null,'saml:AuthnRequestReceivedAt'=>1660912195.505402,'saml:RequestedAuthnContext'=>null,'core:IdP'=>'saml2:https://localhost.someone.from.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/saml2/idp/metadata.php','core:SP'=>'https://pc-example.org.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/module.php/saml/sp/metadata.php/default-sp','IdPMetadata'=>['host'=>'localhost.someone.from.hr','privatekey'=>'key.pem','certificate'=>'cert.pem','auth'=>'example-userpass','attributes.NameFormat'=>'urn:oasis:names:tc:SAML:2.0:attrname-format:uri','authproc'=>[100=>['class'=>'core:AttributeMap',0=>'name2oid',],],'entityid'=>'https://localhost.someone.from.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/saml2/idp/metadata.php','metadata-index'=>'https://localhost.someone.from.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/saml2/idp/metadata.php','metadata-set'=>'saml20-idp-hosted',],'ReturnCallback'=>[0=>'\\SimpleSAML\\IdP',1=>'postAuth',],'Attributes'=>['hrEduPersonUniqueID'=>[0=>'testuser@primjer2.hr',],'urn:oid:0.9.2342.19200300.100.1.1'=>[0=>'testuser',],'urn:oid:2.5.4.3'=>[0=>'TestNameTestSurname',],'urn:oid:2.5.4.4'=>[0=>'TestSurname',1=>'TestSurname2',],'urn:oid:2.5.4.42'=>[0=>'TestName',],'urn:oid:0.9.2342.19200300.100.1.3'=>[0=>'testusermail@primjer.hr',1=>'testusermail2@primjer.hr',],'urn:oid:2.5.4.20'=>[0=>'123',],'hrEduPersonExtensionNumber'=>[0=>'123',],'urn:oid:0.9.2342.19200300.100.1.41'=>[0=>'123',],'urn:oid:2.5.4.23'=>[0=>'123',],'hrEduPersonUniqueNumber'=>[0=>'LOCAL_NO:100',1=>'OIB:00861590360',],'hrEduPersonOIB'=>[0=>'00861590360',],'hrEduPersonDateOfBirth'=>[0=>'20200101',],'hrEduPersonGender'=>[0=>'9',],'urn:oid:0.9.2342.19200300.100.1.60'=>[0=>'987654321',],'urn:oid:2.5.4.36'=>[0=>'cert-value',],'urn:oid:1.3.6.1.4.1.250.1.57'=>[0=>'http://www.org.hr/~ivekHomePage',],'hrEduPersonProfessionalStatus'=>[0=>'VSS',],'hrEduPersonAcademicStatus'=>[0=>'redovitiprofesor',],'hrEduPersonScienceArea'=>[0=>'sociologija',],'hrEduPersonAffiliation'=>[0=>'djelatnik',1=>'student',],'hrEduPersonPrimaryAffiliation'=>[0=>'djelatnik',],'hrEduPersonStudentCategory'=>[0=>'redovitistudent:diplomskisveučilišnistudij',],'hrEduPersonExpireDate'=>[0=>'20211231',],'hrEduPersonTitle'=>[0=>'rektor',],'hrEduPersonRole'=>[0=>'ICTkoordinator',1=>'ISVUkoordinator',],'hrEduPersonStaffCategory'=>[0=>'nastavnoosoblje',1=>'istraživači',],'hrEduPersonGroupMember'=>[0=>'grupa1',1=>'grupa2',],'urn:oid:2.5.4.10'=>[0=>'Testnaustanova',],'hrEduPersonHomeOrg'=>[0=>'primjer.hr',],'urn:oid:2.5.4.11'=>[0=>'Testnaorgjedinica',],'urn:oid:0.9.2342.19200300.100.1.6'=>[0=>'123',],'urn:oid:2.5.4.16'=>[0=>'Testnaustanova,Baštijanova52A,HR-10000Zagreb',],'urn:oid:2.5.4.7'=>[0=>'Zagreb',],'urn:oid:2.5.4.17'=>[0=>'HR-10000',],'urn:oid:2.5.4.9'=>[0=>'Baštijanova52A',],'urn:oid:0.9.2342.19200300.100.1.39'=>[0=>'Testnaadresa52A,HR-10000,Zagreb',],'urn:oid:0.9.2342.19200300.100.1.20'=>[0=>'123',],'hrEduPersonCommURI'=>[0=>'123',],'hrEduPersonPrivacy'=>[0=>'street',1=>'postalCode',],'hrEduPersonPersistentID'=>[0=>'da4294fb4e5746d57ab6ad88d2daf275',],'urn:oid:2.16.840.1.113730.3.1.241'=>[0=>'testname123',],'urn:oid:1.3.6.1.4.1.25178.1.2.12'=>[0=>'skype:pepe.perez',],'hrEduPersonCardNum'=>[0=>'123',],'updatedAt'=>[0=>'123456789',],'urn:oid:1.3.6.1.4.1.5923.1.1.1.7'=>[0=>'urn:example:oidc:manage:client',],],'Authority'=>'example-userpass','AuthnInstant'=>1660911943,'Expire'=>1660940743,'ReturnCall'=>[0=>'\\SimpleSAML\\IdP',1=>'postAuthProc',],'Destination'=>['SingleLogoutService'=>[0=>['Binding'=>'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect','Location'=>'https://pc-example.org.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/module.php/saml/sp/singleLogoutService/default-sp',],],'AssertionConsumerService'=>[0=>['Binding'=>'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST','Location'=>'https://pc-example.org.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/module.php/saml/sp/assertionConsumerService/default-sp','index'=>0,],1=>['Binding'=>'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Artifact','Location'=>'https://pc-example.org.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/module.php/saml/sp/assertionConsumerService/default-sp','index'=>1,],],'contacts'=>[0=>['emailAddress'=>'example@org.hr','givenName'=>'MarkoIvančić','contactType'=>'technical',],],'entityid'=>'https://pc-example.org.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/module.php/saml/sp/metadata.php/default-sp','metadata-index'=>'https://pc-example.org.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/module.php/saml/sp/metadata.php/default-sp','metadata-set'=>'saml20-sp-remote',],'Source'=>['host'=>'localhost.someone.from.hr','privatekey'=>'key.pem','certificate'=>'cert.pem','auth'=>'example-userpass','attributes.NameFormat'=>'urn:oasis:names:tc:SAML:2.0:attrname-format:uri','authproc'=>[100=>['class'=>'core:AttributeMap',0=>'name2oid',],],'entityid'=>'https://localhost.someone.from.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/saml2/idp/metadata.php','metadata-index'=>'https://localhost.someone.from.hr:9074/simplesamlphp/simplesamlphp-2-beta-git/saml2/idp/metadata.php','metadata-set'=>'saml20-idp-hosted',],'\\SimpleSAML\\Auth\\ProcessingChain.filters'=>[],];
-        $state = new State($sampleStateArray);
-
-        $authenticationEvent = new Event($state);
-
-        //$dataStore->persist($authenticationEvent);
-
-        $data = [
-            'jobs_store' => $this->moduleConfiguration->getJobsStoreClass(),
-            'jobs_store_needs_setup' => $jobsStoreNeedsSetup ? 'yes' : 'no',
-            'jobs_store_setup_ran' => $jobsStoreSetupRan ? 'yes' : 'no',
-
-            'data_store' => get_class($dataStore),
-            'data_store_needs_setup' => $dataStoreNeedsSetup ? 'yes' : 'no',
-            'data_store_setup_ran' => $dataStoreSetupRan ? 'yes' : 'no',
-        ];
-
-        die(var_dump($data));
-
-        $template->data = $data;
         return $template;
     }
-
-    public function jobRunner():void
-    {
-        $jobsStore = (new JobsStoreBuilder($this->moduleConfiguration, $this->logger))
-            ->build($this->moduleConfiguration->getJobsStoreClass());
-
-
-        $trackerBuilder = new AuthenticationDataTrackerBuilder($this->moduleConfiguration, $this->logger);
-
-        $configuredTrackers = array_merge(
-            [$this->moduleConfiguration->getDefaultDataTrackerAndProviderClass()],
-            $this->moduleConfiguration->getAdditionalTrackers()
-        );
-
-        $start = new \DateTimeImmutable();
-        $data = [
-            'jobs_processed' => 0,
-            'start_time' => $start->format('Y-m-d H:i:s.u'),
-        ];
-
-        /** @var Event\Job $job */
-        while (($job = $jobsStore->dequeue(Event\Job::class)) !== null) {
-            foreach ($configuredTrackers as $tracker) {
-                ($trackerBuilder->build($tracker))->process($job->getPayload());
-                $data['jobs_processed']++;
-            }
-        }
-
-        $end = new \DateTimeImmutable();
-        $data['end_time'] = $end->format('Y-m-d H:i:s.u');
-        $data['duration'] = $end->diff($start)->format('%s seconds, %f microseconds');
-
-        die(var_dump($data));
-    }
 }

src/Http/Controllers/User/Profile.php

 <?php
 
+declare(strict_types=1);
+
 namespace SimpleSAML\Module\accounting\Http\Controllers\User;
 
 use Psr\Log\LoggerInterface;
@@ -18,9 +20,6 @@ use SimpleSAML\XHTML\Template;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
 
-/**
- * @psalm-suppress all TODO mivanci remove this psalm suppress after testing
- */
 class Profile
 {
     protected ModuleConfiguration $moduleConfiguration;
@@ -58,11 +57,11 @@ class Profile
         $this->defaultAuthenticationSource = $moduleConfiguration->getDefaultAuthenticationSource();
         $this->authSimple = $authSimple ?? new Simple($this->defaultAuthenticationSource, $sspConfiguration, $session);
 
-        $this->authenticationDataProviderBuilder = $authenticationDataProviderBuilder ??
-            new AuthenticationDataProviderBuilder($this->moduleConfiguration, $this->logger);
-
         $this->helpersManager = $helpersManager ?? new HelpersManager();
 
+        $this->authenticationDataProviderBuilder = $authenticationDataProviderBuilder ??
+            new AuthenticationDataProviderBuilder($this->moduleConfiguration, $this->logger, $this->helpersManager);
+
         // Make sure the end user is authenticated.
         $this->authSimple->requireAuth();
     }
@@ -80,7 +79,7 @@ class Profile
         foreach ($this->authSimple->getAttributes() as $name => $value) {
             // Convert attribute names to user-friendly names.
             if (array_key_exists($name, $toNameAttributeMap)) {
-                $name = $toNameAttributeMap[$name];
+                $name = (string)$toNameAttributeMap[$name];
             }
             $normalizedAttributes[$name] = implode('; ', $value);
         }
@@ -114,7 +113,6 @@ class Profile
 
         $page = ($page = (int)$request->query->get('page', 1)) > 0 ? $page : 1;
 
-        // TODO mivanci make maxResults configurable
         $maxResults = 10;
         $firstResult = ($page - 1) * $maxResults;
 
@@ -177,9 +175,9 @@ class Profile
     /** TODO mivanci remove after debugging */
     protected function removeDebugDisplayLimits(): void
     {
-        ini_set('xdebug.var_display_max_depth', -1);
-        ini_set('xdebug.var_display_max_children', -1);
-        ini_set('xdebug.var_display_max_data', -1);
+        ini_set('xdebug.var_display_max_depth', '-1');
+        ini_set('xdebug.var_display_max_children', '-1');
+        ini_set('xdebug.var_display_max_data', '-1');
     }
 
     protected function resolveTemplate(string $template): Template

src/ModuleConfiguration.php

@@ -4,6 +4,7 @@ declare(strict_types=1);
 
 namespace SimpleSAML\Module\accounting;
 
+use DateInterval;
 use Exception;
 use SimpleSAML\Configuration;
 use SimpleSAML\Module\accounting\Exceptions\InvalidConfigurationException;
@@ -35,6 +36,8 @@ class ModuleConfiguration
     public const OPTION_JOB_RUNNER_MAXIMUM_EXECUTION_TIME = 'job_runner_maximum_execution_time';
     public const OPTION_JOB_RUNNER_SHOULD_PAUSE_AFTER_NUMBER_OF_JOBS_PROCESSED =
         'job_runner_should_pause_after_number_of_jobs_processed';
+    public const OPTION_TRACKER_DATA_RETENTION_POLICY = 'tracker_data_retention_policy';
+    public const OPTION_CRON_TAG_FOR_TRACKER_DATA_RETENTION_POLICY = 'cron_tag_for_tracker_data_retention_policy';
 
     /**
      * Contains configuration from module configuration file.
@@ -80,7 +83,7 @@ class ModuleConfiguration
         return $this->getConfiguration()->getString(self::OPTION_JOBS_STORE);
     }
 
-    public function getJobRunnerMaximumExecutionTime(): ?\DateInterval
+    public function getJobRunnerMaximumExecutionTime(): ?DateInterval
     {
         $value = $this->get(self::OPTION_JOB_RUNNER_MAXIMUM_EXECUTION_TIME);
 
@@ -95,7 +98,7 @@ class ModuleConfiguration
         }
 
         try {
-            return new \DateInterval($value);
+            return new DateInterval($value);
         } catch (Throwable $exception) {
             $message = sprintf('Can not create DateInterval instance using value %s as parameter.', $value);
             throw new InvalidConfigurationException($message);
@@ -139,6 +142,10 @@ class ModuleConfiguration
         return $this->getConfiguration()->getArray(self::OPTION_CONNECTIONS_AND_PARAMETERS);
     }
 
+    /**
+     * @return string[]
+     * @psalm-suppress MixedReturnTypeCoercion We specifically check if valid class string are provided.
+     */
     public function getAdditionalTrackers(): array
     {
         return $this->getConfiguration()->getArray(self::OPTION_ADDITIONAL_TRACKERS);
@@ -298,6 +305,13 @@ class ModuleConfiguration
             $errors[] = $exception->getMessage();
         }
 
+        try {
+            $this->validateTrackerDataRetentionPolicy();
+        } catch (Throwable $exception) {
+            $errors[] = $exception->getMessage();
+        }
+
+
         if (!empty($errors)) {
             $message = sprintf('Module configuration validation failed with errors: %s', implode(' ', $errors));
             throw new InvalidConfigurationException($message);
@@ -343,9 +357,6 @@ class ModuleConfiguration
         $errors = [];
 
         // Validate additional trackers
-        /**
-         * @var string $trackerClass
-         */
         foreach ($this->getAdditionalTrackers() as $trackerClass) {
             /** @psalm-suppress DocblockTypeContradiction */
             if (!is_string($trackerClass)) {
@@ -444,4 +455,34 @@ class ModuleConfiguration
     {
         $this->getCronTagForJobRunner();
     }
+
+    protected function validateTrackerDataRetentionPolicy(): void
+    {
+        if ($this->getTrackerDataRetentionPolicy() !== null) {
+            $this->getCronTagForTrackerDataRetentionPolicy();
+        }
+    }
+
+    public function getTrackerDataRetentionPolicy(): ?DateInterval
+    {
+        /** @var string|null $value */
+        $value = $this->getConfiguration()
+            ->getOptionalString(self::OPTION_TRACKER_DATA_RETENTION_POLICY, null);
+
+        if (is_null($value)) {
+            return null;
+        }
+
+        try {
+            return new DateInterval($value);
+        } catch (Throwable $exception) {
+            $message = sprintf('Can not create DateInterval instance using value %s as parameter.', $value);
+            throw new InvalidConfigurationException($message);
+        }
+    }
+
+    public function getCronTagForTrackerDataRetentionPolicy(): string
+    {
+        return $this->getConfiguration()->getString(self::OPTION_CRON_TAG_FOR_TRACKER_DATA_RETENTION_POLICY);
+    }
 }

src/Services/JobRunner.php

@@ -493,7 +493,6 @@ class JobRunner
             $this->moduleConfiguration->getAdditionalTrackers()
         );
 
-        /** @var string $trackerClass */
         foreach ($configuredTrackerClasses as $trackerClass) {
             $trackers[$trackerClass] = $this->authenticationDataTrackerBuilder->build($trackerClass);
         }

src/Services/JobRunner/State.php

@@ -15,6 +15,9 @@ class State
     protected ?\DateTimeImmutable $endedAt = null;
     protected int $successfulJobsProcessed = 0;
     protected int $failedJobsProcessed = 0;
+    /**
+     * @var string[]
+     */
     protected array $statusMessages = [];
     protected int $numberOfStatusMessagesToKeep = 10;
     protected bool $isGracefulInterruptInitiated = false;
@@ -161,6 +164,9 @@ class State
         }
     }
 
+    /**
+     * @return string[]
+     */
     public function getStatusMessages(): array
     {
         return $this->statusMessages;
@@ -172,7 +178,7 @@ class State
             return null;
         }
 
-        $message = (string)end($this->statusMessages);
+        $message = end($this->statusMessages);
         reset($this->statusMessages);
 
         return $message;

src/Stores/Data/Authentication/DoctrineDbal/Versioned/Store.php

@@ -506,76 +506,6 @@ class Store extends AbstractStore implements DataStoreInterface
         }
 
         return $connectedServiceProviderBag;
-
-        // TODO mivanci remove after unit tests
-//        $authenticationEventsQueryBuilder = $this->connection->dbal()->createQueryBuilder();
-//        $lastMetadataAndAttributesQueryBuilder = $this->connection->dbal()->createQueryBuilder();
-//
-//        /** @psalm-suppress TooManyArguments */
-//        $authenticationEventsQueryBuilder->select(
-//            'vs.entity_id AS sp_entity_id',
-//            'COUNT(vae.id) AS number_of_authentications',
-//            'MAX(vae.happened_at) AS last_authentication_at',
-//            'MIN(vae.happened_at) AS first_authentication_at',
-//        )->from('vds_authentication_event', 'vae')
-//            ->leftJoin(
-//                'vae',
-//                'vds_idp_sp_user_version',
-//                'visuv',
-//                'vae.idp_sp_user_version_id = visuv.id'
-//            )
-//            ->leftJoin('visuv', 'vds_sp_version', 'vsv', 'visuv.sp_version_id = vsv.id')
-//            ->leftJoin('vsv', 'vds_sp', 'vs', 'vsv.sp_id = vs.id')
-//            ->leftJoin('visuv', 'vds_user_version', 'vuv', 'visuv.user_version_id = vuv.id')
-//            ->leftJoin('vuv', 'vds_user', 'vu', 'vuv.user_id = vu.id')
-//            ->where(
-//                'vu.identifier_hash_sha256 = ' .
-//                $authenticationEventsQueryBuilder->createNamedParameter($userIdentifierHashSha256)
-//            )
-//            ->groupBy('vs.id')
-//            ->orderBy('number_of_authentications', 'DESC');
-//
-//        /** @psalm-suppress TooManyArguments */
-//        $lastMetadataAndAttributesQueryBuilder->select(
-//            'vs.entity_id AS sp_entity_id',
-//            'vsv.metadata AS sp_metadata',
-//            'vuv.attributes AS user_attributes',
-//            //            'vsv.id AS sp_version_id',
-//            //            'vuv.id AS user_version_id',
-//        )->from('vds_authentication_event', 'vae')
-//            ->leftJoin(
-//                'vae',
-//                'vds_idp_sp_user_version',
-//                'visuv',
-//                'vae.idp_sp_user_version_id = visuv.id'
-//            )
-//            ->leftJoin('visuv', 'vds_sp_version', 'vsv', 'visuv.sp_version_id = vsv.id')
-//            ->leftJoin('vsv', 'vds_sp', 'vs', 'vsv.sp_id = vs.id')
-//            ->leftJoin('visuv', 'vds_user_version', 'vuv', 'visuv.user_version_id = vuv.id')
-//            ->leftJoin('vuv', 'vds_user', 'vu', 'vuv.user_id = vu.id')
-//            ->leftJoin('vsv', 'vds_sp_version', 'vsv2', 'vsv.id = vsv2.id AND vsv.id < vsv2.id')
-//            ->leftJoin('vuv', 'vds_user_version', 'vuv2', 'vuv.id = vuv2.id AND vuv.id < vuv2.id')
-//            ->where(
-//                'vu.identifier_hash_sha256 = ' .
-//                $lastMetadataAndAttributesQueryBuilder->createNamedParameter($userIdentifierHashSha256)
-//            )
-//            ->andWhere('vsv2.id IS NULL')
-//            ->andWhere('vuv2.id IS NULL');
-//
-//        try {
-//            $numberOfAuthentications =
-// $authenticationEventsQueryBuilder->executeQuery()->fetchAllAssociativeIndexed();
-//            $lastMetadataAndAttributes =
-//                $lastMetadataAndAttributesQueryBuilder->executeQuery()->fetchAllAssociativeIndexed();
-//
-//            return array_merge_recursive($numberOfAuthentications, $lastMetadataAndAttributes);
-//        } catch (\Throwable $exception) {
-//            $message = sprintf(
-//                'Error executing query to get connected organizations. Error was: %s.',
-//                $exception->getMessage()
-//            );
-//            throw new StoreException($message, (int)$exception->getCode(), $exception);
-//        }
     }
 
 
@@ -584,7 +514,6 @@ class Store extends AbstractStore implements DataStoreInterface
      */
     public function getActivity(string $userIdentifierHashSha256, int $maxResults, int $firstResult): Activity\Bag
     {
-        // TODO mivanci pagination
         $results =  $this->repository->getActivity($userIdentifierHashSha256, $maxResults, $firstResult);
 
         $activityBag = new Activity\Bag();
@@ -618,42 +547,11 @@ class Store extends AbstractStore implements DataStoreInterface
         }
 
         return $activityBag;
+    }
 
-        // TODO mivanci remove
-//        $authenticationEventsQueryBuilder = $this->connection->dbal()->createQueryBuilder();
-//
-//        /** @psalm-suppress TooManyArguments */
-//        $authenticationEventsQueryBuilder->select(
-//            'vae.happened_at',
-//            'vsv.metadata AS sp_metadata',
-//            'vuv.attributes AS user_attributes'
-//        )->from('vds_authentication_event', 'vae')
-//            ->leftJoin(
-//                'vae',
-//                'vds_idp_sp_user_version',
-//                'visuv',
-//                'vae.idp_sp_user_version_id = visuv.id'
-//            )
-//            ->leftJoin('visuv', 'vds_sp_version', 'vsv', 'visuv.sp_version_id = vsv.id')
-//            ->leftJoin('vsv', 'vds_sp', 'vs', 'vsv.sp_id = vs.id')
-//            ->leftJoin('visuv', 'vds_user_version', 'vuv', 'visuv.user_version_id = vuv.id')
-//            ->leftJoin('vuv', 'vds_user', 'vu', 'vuv.user_id = vu.id')
-//            ->where(
-//                'vu.identifier_hash_sha256 = ' .
-//                $authenticationEventsQueryBuilder->createNamedParameter($userIdentifierHashSha256)
-//            )
-//            ->orderBy('vae.id', 'DESC');
-//
-//        try {
-//            $numberOfAuthentications = $authenticationEventsQueryBuilder->executeQuery()->fetchAllAssociative();
-//
-//            return array_merge_recursive($numberOfAuthentications);
-//        } catch (\Throwable $exception) {
-//            $message = sprintf(
-//                'Error executing query to get connected organizations. Error was: %s.',
-//                $exception->getMessage()
-//            );
-//            throw new StoreException($message, (int)$exception->getCode(), $exception);
-//        }
+    public function deleteDataOlderThan(\DateTimeImmutable $dateTime): void
+    {
+        // Only delete authentication events. Versioned data (IdP / SP metadata, user attributes) remain.
+        $this->repository->deleteAuthenticationEventsOlderThan($dateTime);
     }
 }

src/Stores/Data/Authentication/DoctrineDbal/Versioned/Store/Migrations/Version20220801000700CreateAuthenticationEventTable.php

@@ -50,6 +50,9 @@ class Version20220801000700CreateAuthenticationEventTable extends AbstractMigrat
                 ['id']
             );
 
+            // Old data can be deleted using happened_at column, so add index for it.
+            $table->addIndex(['happened_at']);
+
             $this->schemaManager->createTable($table);
         } catch (\Throwable $exception) {
             throw $this->prepareGenericMigrationException(

src/Stores/Data/Authentication/DoctrineDbal/Versioned/Store/Repository.php

@@ -1066,4 +1066,28 @@ class Repository
             throw new StoreException($message, (int)$exception->getCode(), $exception);
         }
     }
+
+    /**
+     * @throws StoreException
+     */
+    public function deleteAuthenticationEventsOlderThan(\DateTimeImmutable $dateTime): void
+    {
+        try {
+            $queryBuilder = $this->connection->dbal()->createQueryBuilder();
+
+            $queryBuilder->delete($this->tableNameAuthenticationEvent)
+                ->where(
+                    $queryBuilder->expr()->lt(
+                        TableConstants::TABLE_AUTHENTICATION_EVENT_COLUMN_NAME_HAPPENED_AT,
+                        $queryBuilder->createNamedParameter($dateTime, Types::DATETIME_IMMUTABLE)
+                    )
+                )->executeStatement();
+        } catch (Throwable $exception) {
+            $message = sprintf(
+                'Error executing query to delete old authentication events. Error was: %s.',
+                $exception->getMessage()
+            );
+            throw new StoreException($message, (int)$exception->getCode(), $exception);
+        }
+    }
 }

src/Stores/Interfaces/DataStoreInterface.php

@@ -24,4 +24,6 @@ interface DataStoreInterface extends StoreInterface
     public function getConnectedOrganizations(string $userIdentifierHashSha256): ConnectedServiceProvider\Bag;
 
     public function getActivity(string $userIdentifierHashSha256, int $maxResults, int $firstResult): Activity\Bag;
+
+    public function deleteDataOlderThan(\DateTimeImmutable $dateTime): void;
 }

src/Stores/Jobs/PhpRedis/RedisStore.php

@@ -137,6 +137,7 @@ class RedisStore extends AbstractStore implements JobsStoreInterface
 
     /**
      * @throws StoreException
+     * @codeCoverageIgnore
      */
     public static function build(
         ModuleConfiguration $moduleConfiguration,