Infrastructure

Index task processor was unresponsive and not reporting new events to the logs. Error:

Aug 23, 2018 3:33:15 AM com.hazelcast.client.ProxyHelper
INFO: There is no response for Call [8081307] operation=CONCURRENT_MAP_GET in 5 seconds.
java.lang.NullPointerException
    at java.net.URI$Parser.parse(URI.java:3042)
    at java.net.URI.<init>(URI.java:588)
    at java.net.URI.create(URI.java:850)
    at org.apache.http.client.methods.HttpGet.<init>(HttpGet.java:69)
    at org.dataone.cn.indexer.convert.MemberNodeServiceRegistrationTypeDocumentService.fetchServiceTypeDoc(MemberNodeServiceRegistrationTypeDocumentService.java:74)
    at org.dataone.cn.indexer.convert.MemberNodeServiceRegistrationTypeDocumentService.getMemberNodeServiceRegistrationTypeDocument(MemberNodeServiceRegistrationTypeDocumentService.java:57)
    at org.dataone.cn.indexer.parser.utility.MemberNodeServiceRegistrationTypeCache.getServiceTypes(MemberNodeServiceRegistrationTypeCache.java:52)
    at org.dataone.cn.indexer.convert.MemberNodeServiceRegistrationTypeConverter.convert(MemberNodeServiceRegistrationTypeConverter.java:38)
    at org.dataone.cn.indexer.parser.SolrField.processNodeValue(SolrField.java:216)
    at org.dataone.cn.indexer.parser.SolrField.processField(SolrField.java:162)
    at org.dataone.cn.indexer.parser.SolrField.getFields(SolrField.java:119)
    at org.dataone.cn.indexer.parser.BaseXPathDocumentSubprocessor.processDocument(BaseXPathDocumentSubprocessor.java:114)
    at org.dataone.cn.indexer.parser.ScienceMetadataDocumentSubprocessor.processDocument(ScienceMetadataDocumentSubprocessor.java:42)
    at org.dataone.cn.indexer.SolrIndexService.processObject(SolrIndexService.java:230)
    at org.dataone.cn.indexer.SolrIndexService.insertIntoIndex(SolrIndexService.java:303)
    at org.dataone.cn.index.processor.IndexTaskUpdateProcessor.process(IndexTaskUpdateProcessor.java:50)
    at org.dataone.cn.index.processor.IndexTaskProcessor.processTask(IndexTaskProcessor.java:286)
    at org.dataone.cn.index.processor.IndexTaskProcessor.access$000(IndexTaskProcessor.java:80)
    at org.dataone.cn.index.processor.IndexTaskProcessor$1.run(IndexTaskProcessor.java:263)
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at java.lang.Thread.run(Thread.java:748)
java.lang.NullPointerException
    at java.net.URI$Parser.parse(URI.java:3042)
    at java.net.URI.<init>(URI.java:588)
    at java.net.URI.create(URI.java:850)
    at org.apache.http.client.methods.HttpGet.<init>(HttpGet.java:69)
    at org.dataone.cn.indexer.convert.MemberNodeServiceRegistrationTypeDocumentService.fetchServiceTypeDoc(MemberNodeServiceRegistrationTypeDocumentService.java:74)
    at org.dataone.cn.indexer.convert.MemberNodeServiceRegistrationTypeDocumentService.getMemberNodeServiceRegistrationTypeDocument(MemberNodeServiceRegistrationTypeDocumentService.java:57)
    at org.dataone.cn.indexer.parser.utility.MemberNodeServiceRegistrationTypeCache.getServiceTypes(MemberNodeServiceRegistrationTypeCache.java:52)
    at org.dataone.cn.indexer.convert.MemberNodeServiceRegistrationTypeConverter.convert(MemberNodeServiceRegistrationTypeConverter.java:38)
    at org.dataone.cn.indexer.parser.SolrField.processNodeValue(SolrField.java:216)
    at org.dataone.cn.indexer.parser.SolrField.processField(SolrField.java:162)
    at org.dataone.cn.indexer.parser.SolrField.getFields(SolrField.java:119)
    at org.dataone.cn.indexer.parser.BaseXPathDocumentSubprocessor.processDocument(BaseXPathDocumentSubprocessor.java:114)
    at org.dataone.cn.indexer.parser.ScienceMetadataDocumentSubprocessor.processDocument(ScienceMetadataDocumentSubprocessor.java:42)
    at org.dataone.cn.indexer.SolrIndexService.processObject(SolrIndexService.java:230)
    at org.dataone.cn.indexer.SolrIndexService.insertIntoIndex(SolrIndexService.java:303)
    at org.dataone.cn.index.processor.IndexTaskUpdateProcessor.process(IndexTaskUpdateProcessor.java:50)
    at org.dataone.cn.index.processor.IndexTaskProcessor.processTask(IndexTaskProcessor.java:286)
    at org.dataone.cn.index.processor.IndexTaskProcessor.access$000(IndexTaskProcessor.java:80)
    at org.dataone.cn.index.processor.IndexTaskProcessor$1.run(IndexTaskProcessor.java:263)
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at java.lang.Thread.run(Thread.java:748)

Service restarted and continued processing OK.

Need to diagnose why the service stopped and how to prevent in the future.