ConcurrentModificationException when iterating over validation keys

[tloodu]

Describe the bug

When running the ltpa_fat FAT suite, testDifferentDirectoriesForValidationKeys() in the ltpaKeyRotationTests class fails with a ConcurrentModificationException occurring in the validateTokenBytes() function of LTPAToken2Factory.

This CME occurs during the iteration process when validationKeys is being checked until a token is successfully validated. The implemented solution is safe and should work validationKeysIterator.remove(), however if validationKeys is also modified (by another thread) externally of the iterator then a CME is thrown.

Stack Trace

>Exception = java.util.ConcurrentModificationException
>Source = com.ibm.ws.security.authentication.jaas.modules.TokenLoginModule
>probeid = 118
>Stack Dump = java.util.ConcurrentModificationException
>	at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:922)
>	at java.util.ArrayList$Itr.next(ArrayList.java:872)
>	at com.ibm.ws.security.token.ltpa.internal.LTPAToken2Factory.validateTokenBytes(LTPAToken2Factory.java:121)
>	at com.ibm.ws.security.token.ltpa.internal.LTPATokenService.recreateTokenFromBytes(LTPATokenService.java:75)
>	at com.ibm.ws.security.token.internal.TokenManagerImpl.recreateTokenFromBytes(TokenManagerImpl.java:127)
>	at com.ibm.ws.security.authentication.jaas.modules.TokenLoginModule.login(TokenLoginModule.java:102)
>	at com.ibm.ws.kernel.boot.security.LoginModuleProxy.login(LoginModuleProxy.java:53)
>	at sun.reflect.GeneratedMethodAccessor12.invoke(Unknown Source)
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:55)
>	at java.lang.reflect.Method.invoke(Method.java:508)
>	at javax.security.auth.login.LoginContext.invoke(LoginContext.java:788)
>	at javax.security.auth.login.LoginContext.access$000(LoginContext.java:196)
>	at javax.security.auth.login.LoginContext$4.run(LoginContext.java:698)
>	at javax.security.auth.login.LoginContext$4.run(LoginContext.java:696)
>	at java.security.AccessController.doPrivileged(AccessController.java:746)
>	at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:696)
>	at javax.security.auth.login.LoginContext.login(LoginContext.java:597)
>	at com.ibm.ws.security.authentication.internal.jaas.JAASServiceImpl.doLoginContext(JAASServiceImpl.java:369)
>	at com.ibm.ws.security.authentication.internal.jaas.JAASServiceImpl.performLogin(JAASServiceImpl.java:355)
>	at com.ibm.ws.security.authentication.internal.jaas.JAASServiceImpl.performLogin(JAASServiceImpl.java:340)
>	at com.ibm.ws.security.authentication.internal.AuthenticationServiceImpl.performJAASLogin(AuthenticationServiceImpl.java:569)
>	at com.ibm.ws.security.authentication.internal.AuthenticationServiceImpl.authenticate(AuthenticationServiceImpl.java:228)
>	at com.ibm.ws.webcontainer.security.internal.SSOAuthenticator.handleLtpaSSO(SSOAuthenticator.java:185)
>	at com.ibm.ws.webcontainer.security.internal.SSOAuthenticator.handleSSO(SSOAuthenticator.java:139)
>	at com.ibm.ws.webcontainer.security.internal.SSOAuthenticator.authenticate(SSOAuthenticator.java:98)
>	at com.ibm.ws.webcontainer.security.internal.SSOAuthenticator.authenticate(SSOAuthenticator.java:88)
>	at com.ibm.ws.webcontainer.security.WebProviderAuthenticatorProxy.handleSSO(WebProviderAuthenticatorProxy.java:351)
>	at com.ibm.ws.webcontainer.security.WebProviderAuthenticatorProxy.authenticate(WebProviderAuthenticatorProxy.java:467)
>	at com.ibm.ws.webcontainer.security.WebAuthenticatorProxy.authenticate(WebAuthenticatorProxy.java:69)
>	at com.ibm.ws.webcontainer.security.WebAppSecurityCollaboratorImpl.authenticateRequest(WebAppSecurityCollaboratorImpl.java:1239)
>	at com.ibm.ws.webcontainer.security.WebAppSecurityCollaboratorImpl.determineWebReply(WebAppSecurityCollaboratorImpl.java:995)
>	at com.ibm.ws.webcontainer.security.WebAppSecurityCollaboratorImpl.performSecurityChecks(WebAppSecurityCollaboratorImpl.java:695)
>	at com.ibm.ws.webcontainer.security.WebAppSecurityCollaboratorImpl.preInvoke(WebAppSecurityCollaboratorImpl.java:622)
>	at com.ibm.wsspi.webcontainer.collaborator.CollaboratorHelper.preInvokeCollaborators(CollaboratorHelper.java:472)
>	at com.ibm.ws.webcontainer.osgi.collaborator.CollaboratorHelperImpl.preInvokeCollaborators(CollaboratorHelperImpl.java:353)
>	at com.ibm.ws.webcontainer.filter.WebAppFilterManager.invokeFilters(WebAppFilterManager.java:1222)
>	at com.ibm.ws.webcontainer.filter.WebAppFilterManager.invokeFilters(WebAppFilterManager.java:1078)
>	at com.ibm.ws.webcontainer.servlet.CacheServletWrapper.handleRequest(CacheServletWrapper.java:77)
>	at com.ibm.ws.webcontainer.WebContainer.handleRequest(WebContainer.java:978)
>	at com.ibm.ws.webcontainer.osgi.DynamicVirtualHost$2.run(DynamicVirtualHost.java:293)
>	at com.ibm.ws.http.dispatcher.internal.channel.HttpDispatcherLink$TaskWrapper.run(HttpDispatcherLink.java:1284)
>	at com.ibm.ws.http.dispatcher.internal.channel.HttpDispatcherLink.wrapHandlerAndExecute(HttpDispatcherLink.java:500)
>	at com.ibm.ws.http.dispatcher.internal.channel.HttpDispatcherLink.ready(HttpDispatcherLink.java:459)
>	at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.handleDiscrimination(HttpInboundLink.java:569)
>	at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.handleNewRequest(HttpInboundLink.java:503)
>	at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.processRequest(HttpInboundLink.java:363)
>	at com.ibm.ws.http.channel.internal.inbound.HttpICLReadCallback.complete(HttpICLReadCallback.java:72)
>	at com.ibm.ws.tcpchannel.internal.WorkQueueManager.requestComplete(WorkQueueManager.java:516)
>	at com.ibm.ws.tcpchannel.internal.WorkQueueManager.attemptIO(WorkQueueManager.java:586)
>	at com.ibm.ws.tcpchannel.internal.WorkQueueManager.workerRun(WorkQueueManager.java:970)
>	at com.ibm.ws.tcpchannel.internal.WorkQueueManager$Worker.run(WorkQueueManager.java:1059)
>	at com.ibm.ws.threading.internal.ExecutorServiceImpl$RunnableWrapper.run(ExecutorServiceImpl.java:298)
>	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160)
>	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
>	at java.lang.Thread.run(Thread.java:825)
]
	at componenttest.custom.junit.runner.FATRunner.blowup(FATRunner.java:363)
	at componenttest.custom.junit.runner.FATRunner.access$600(FATRunner.java:71)
	at componenttest.custom.junit.runner.FATRunner$1.evaluate(FATRunner.java:302)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:263)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:68)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:47)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:30)
	at componenttest.custom.junit.runner.FATRunner$2.evaluate(FATRunner.java:381)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:300)
	at componenttest.custom.junit.runner.FATRunner.run(FATRunner.java:185)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:24)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:300)
	at junit.framework.JUnit4TestAdapter.run(JUnit4TestAdapter.java:39)
	at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(JUnitTestRunner.java:520)
	at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.launch(JUnitTestRunner.java:1060)
	at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.main(JUnitTestRunner.java:911)

Steps to Reproduce
This is an intermittent issue that does not occur every time as it is a concurrency related problem.

Expected behavior
The test should pass and the iterator should retrieve elements without a CME.

Diagnostic information:

• Java Version: IBM Java JDK 8
• LTPA FAT Server Logs
• FAT Results