java.util.ConcurrentModificationException if multiple stages in a pipeline archive reports

[adamdougal]

I'm sometimes getting the following exception when running a parallel pipeline:

java.util.ConcurrentModificationException
	at java.util.HashMap$HashIterator.nextNode(HashMap.java:1445)
	at java.util.HashMap$EntryIterator.next(HashMap.java:1479)
	at java.util.HashMap$EntryIterator.next(HashMap.java:1477)
	at com.thoughtworks.xstream.converters.collections.MapConverter.marshal(MapConverter.java:75)
	at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
	at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
	at com.thoughtworks.xstream.core.AbstractReferenceMarshaller$1.convertAnother(AbstractReferenceMarshaller.java:84)
	at hudson.util.RobustReflectionConverter.marshallField(RobustReflectionConverter.java:263)
	at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:250)
Caused: java.lang.RuntimeException: Failed to serialize net.masterthought.jenkins.SafeArchiveServingAction#fileChecksums for class net.masterthought.jenkins.SafeArchiveServingRunAction
	at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:254)
	at hudson.util.RobustReflectionConverter$2.visit(RobustReflectionConverter.java:222)
	at com.thoughtworks.xstream.converters.reflection.PureJavaReflectionProvider.visitSerializableFields(PureJavaReflectionProvider.java:138)
	at hudson.util.RobustReflectionConverter.doMarshal(RobustReflectionConverter.java:208)
	at hudson.util.RobustReflectionConverter.marshal(RobustReflectionConverter.java:149)
	at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
	at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
	at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:43)
	at com.thoughtworks.xstream.core.AbstractReferenceMarshaller$1.convertAnother(AbstractReferenceMarshaller.java:88)
	at com.thoughtworks.xstream.converters.collections.AbstractCollectionConverter.writeItem(AbstractCollectionConverter.java:64)
	at com.thoughtworks.xstream.converters.collections.CollectionConverter.marshal(CollectionConverter.java:74)
	at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
	at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
	at com.thoughtworks.xstream.core.AbstractReferenceMarshaller$1.convertAnother(AbstractReferenceMarshaller.java:84)
	at hudson.util.RobustReflectionConverter.marshallField(RobustReflectionConverter.java:263)
	at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:250)
Caused: java.lang.RuntimeException: Failed to serialize hudson.model.Actionable#actions for class org.jenkinsci.plugins.workflow.job.WorkflowRun
	at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:254)
	at hudson.util.RobustReflectionConverter$2.visit(RobustReflectionConverter.java:222)
	at com.thoughtworks.xstream.converters.reflection.PureJavaReflectionProvider.visitSerializableFields(PureJavaReflectionProvider.java:138)
	at hudson.util.RobustReflectionConverter.doMarshal(RobustReflectionConverter.java:208)
	at hudson.util.RobustReflectionConverter.marshal(RobustReflectionConverter.java:149)
	at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
	at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
	at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:43)
	at com.thoughtworks.xstream.core.TreeMarshaller.start(TreeMarshaller.java:82)
	at com.thoughtworks.xstream.core.AbstractTreeMarshallingStrategy.marshal(AbstractTreeMarshallingStrategy.java:37)
	at com.thoughtworks.xstream.XStream.marshal(XStream.java:1026)
	at com.thoughtworks.xstream.XStream.marshal(XStream.java:1015)
	at com.thoughtworks.xstream.XStream.toXML(XStream.java:988)
	at hudson.util.XStream2.toXMLUTF8(XStream2.java:313)
	at org.jenkinsci.plugins.workflow.support.PipelineIOUtils.writeByXStream(PipelineIOUtils.java:34)
	at org.jenkinsci.plugins.workflow.job.WorkflowRun.save(WorkflowRun.java:1158)
	at hudson.BulkChange.commit(BulkChange.java:98)
	at org.jenkinsci.plugins.workflow.cps.CpsFlowExecution.notifyListeners(CpsFlowExecution.java:1475)
	at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$3.run(CpsThreadGroup.java:458)
	at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$1.run(CpsVmExecutorService.java:38)
	at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:131)
	at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28)
	at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:59)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

I think it's failing at the point of archiving cucumber reports, possibly because multiple stages are trying to do it at the same time. If I run the following pipeline I get fairly consistent failures

parallel 'Functional Test': {
    stage('Run Functional Tests') {
        node('some-slave') {
            deleteDir()
            dir('some-dir') {
                checkout([$class           : 'GitSCM', branches: [[name: 'master']],
                   userRemoteConfigs: [[credentialsId: 'some-creds', url: 'some-repo.git']]])

                if (findFiles(glob: '**/cucumber-reports/*.json')) {
                    cucumber fileIncludePattern: '**/cucumber-reports/*.json'
                }
            }
        }
    }
}, 'Stubbed NFT': {
    stage('Run Stubbed NFT') {
        node('some-slave') {
            deleteDir()
            dir('some-dir') {
                checkout([$class           : 'GitSCM', branches: [[name: 'master']],
                   userRemoteConfigs: [[credentialsId: 'some-creds', url: 'some-repo.git']]])

                if (findFiles(glob: '**/cucumber-reports/*.json')) {
                    cucumber fileIncludePattern: '**/cucumber-reports/*.json'
                }
            }
        }
    }
}, 'Integrated Test': {
    stage('Run Integrated Tests') {
        node('some-slave') {
            deleteDir()
            dir('some-dir') {
                checkout([$class           : 'GitSCM', branches: [[name: 'master']],
                   userRemoteConfigs: [[credentialsId: 'some-creds', url: 'some-repo.git']]])

                if (findFiles(glob: '**/cucumber-reports/*.json')) {
                    cucumber fileIncludePattern: '**/cucumber-reports/*.json'
                }
            }
        }
    }
}

[damianszczepanik]

@daniel-beck have you seen something like that?

[suganyaravikumar]

We observe a similar error in our parallel test stages frequently and it seems to be in the GitSCM checkout step.

• In blue ocean pipeline GIT SCM checkout step
  ....
  using GIT_ASKPASS to set credentials

git lfs pull origin # timeout=30
Failed to serialize org.jenkinsci.plugins.workflow.job.WorkflowRun#execution for class org.jenkinsci.plugins.workflow.job.WorkflowRun

• In console log
  java.util.ConcurrentModificationException
  at java.util.HashMap$HashIterator.nextNode(HashMap.java:1445)
  at java.util.HashMap$EntryIterator.next(HashMap.java:1479)
  at java.util.HashMap$EntryIterator.next(HashMap.java:1477)
  at com.thoughtworks.xstream.converters.collections.MapConverter.marshal(MapConverter.java:75)
  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:43)
  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller$1.convertAnother(AbstractReferenceMarshaller.java:88)
  at org.jenkinsci.plugins.workflow.cps.CpsFlowExecution$ConverterImpl.writeChild(CpsFlowExecution.java:1683)
  at org.jenkinsci.plugins.workflow.cps.CpsFlowExecution$ConverterImpl.marshal(CpsFlowExecution.java:1628)
  at hudson.util.XStream2$AssociatedConverterImpl.marshal(XStream2.java:460)
  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller$1.convertAnother(AbstractReferenceMarshaller.java:84)
  at hudson.util.RobustReflectionConverter.marshallField(RobustReflectionConverter.java:263)
  at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:250)
  Caused: java.lang.RuntimeException: Failed to serialize org.jenkinsci.plugins.workflow.job.WorkflowRun#execution for class org.jenkinsci.plugins.workflow.job.WorkflowRun
  at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:254)
  at hudson.util.RobustReflectionConverter$2.visit(RobustReflectionConverter.java:222)
  at com.thoughtworks.xstream.converters.reflection.PureJavaReflectionProvider.visitSerializableFields(PureJavaReflectionProvider.java:138)
  at hudson.util.RobustReflectionConverter.doMarshal(RobustReflectionConverter.java:208)
  at hudson.util.RobustReflectionConverter.marshal(RobustReflectionConverter.java:149)
  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:43)
  at com.thoughtworks.xstream.core.TreeMarshaller.start(TreeMarshaller.java:82)
  at com.thoughtworks.xstream.core.AbstractTreeMarshallingStrategy.marshal(AbstractTreeMarshallingStrategy.java:37)
  at com.thoughtworks.xstream.XStream.marshal(XStream.java:1026)
  at com.thoughtworks.xstream.XStream.marshal(XStream.java:1015)
  at com.thoughtworks.xstream.XStream.toXML(XStream.java:988)
  at hudson.util.XStream2.toXMLUTF8(XStream2.java:313)
  at org.jenkinsci.plugins.workflow.support.PipelineIOUtils.writeByXStream(PipelineIOUtils.java:34)
  at org.jenkinsci.plugins.workflow.job.WorkflowRun.save(WorkflowRun.java:1158)
  at hudson.util.PersistedList.onModified(PersistedList.java:173)
  at hudson.util.PersistedList._onModified(PersistedList.java:181)
  at hudson.util.PersistedList.add(PersistedList.java:72)
  at org.jenkinsci.plugins.workflow.job.WorkflowRun.onCheckout(WorkflowRun.java:856)
  at org.jenkinsci.plugins.workflow.job.WorkflowRun.access$1000(WorkflowRun.java:133)
  at org.jenkinsci.plugins.workflow.job.WorkflowRun$SCMListenerImpl.onCheckout(WorkflowRun.java:1116)
  at org.jenkinsci.plugins.workflow.steps.scm.SCMStep.checkout(SCMStep.java:140)
  at org.jenkinsci.plugins.workflow.steps.scm.SCMStep$StepExecutionImpl.run(SCMStep.java:93)
  at org.jenkinsci.plugins.workflow.steps.scm.SCMStep$StepExecutionImpl.run(SCMStep.java:80)
  at org.jenkinsci.plugins.workflow.steps.SynchronousNonBlockingStepExecution.lambda$start$0(SynchronousNonBlockingStepExecution.java:47)
  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
  at java.util.concurrent.FutureTask.run(FutureTask.java:266)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  at java.lang.Thread.run(Thread.java:748)