cassandra benchmark ReadWriteTest can not run successfully - cassandra
I run cassandra benchmarks ReadWriteTest with idea on centos7
`
package org.apache.cassandra.test.microbench;
#BenchmarkMode(Mode.Throughput)
#OutputTimeUnit(TimeUnit.MILLISECONDS)
#Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
#Measurement(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
#Fork(value = 1)
#Threads(1)
#State(Scope.Benchmark)
public class ReadWriteTest extends CQLTester
{
static String keyspace;
String table;
String writeStatement;
String readStatement;
long numRows = 0;
ColumnFamilyStore cfs;
#Setup(Level.Trial)
public void setup() throws Throwable
{
CQLTester.setUpClass();
keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid))");
execute("use "+keyspace+";");
writeStatement = "INSERT INTO "+table+"(userid,picid,commentid)VALUES(?,?,?)";
readStatement = "SELECT * from "+table+" limit 100";
cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
cfs.disableAutoCompaction();
//Warm up
System.err.println("Writing 50k");
for (long i = 0; i < 5000; i++)
execute(writeStatement, i, i, i );
}
#TearDown(Level.Trial)
public void teardown() throws IOException, ExecutionException, InterruptedException
{
CQLTester.cleanup();
}
#Benchmark
public Object write() throws Throwable
{
numRows++;
return execute(writeStatement, numRows, numRows, numRows );
}
#Benchmark
public Object read() throws Throwable
{
return execute(readStatement);
}
public static void main(String[] args) throws RunnerException {
Options opt = new OptionsBuilder()
.include(ReadWriteTest.class.getSimpleName())
.build();
new Runner(opt).run();
}
}
it seems to be successful at start, "java.lang.IndexOutOfBoundsException: Index: 0, Size: 0" occurs like this
01:37:25.884 [org.apache.cassandra.test.microbench.ReadWriteTest.read-jmh-worker-1] DEBUG org.apache.cassandra.db.ReadCommand - --in queryMemtableAndDiskInternal, partitionKey token:-2532556674411782010, partitionKey:DecoratedKey(-2532556674411782010, 6b657973706163655f30)
01:37:25.888 [org.apache.cassandra.test.microbench.ReadWriteTest.read-jmh-worker-1] INFO o.a.cassandra.db.ColumnFamilyStore - Initializing keyspace_0.globalReplicaTable
01:37:25.889 [org.apache.cassandra.test.microbench.ReadWriteTest.read-jmh-worker-1] DEBUG o.a.cassandra.db.DiskBoundaryManager - Refreshing disk boundary cache for keyspace_0.globalReplicaTable
01:37:25.890 [org.apache.cassandra.test.microbench.ReadWriteTest.read-jmh-worker-1] DEBUG o.a.cassandra.db.DiskBoundaryManager - Got local ranges [] (ringVersion = 0)
01:37:25.890 [org.apache.cassandra.test.microbench.ReadWriteTest.read-jmh-worker-1] DEBUG o.a.cassandra.db.DiskBoundaryManager - Updating boundaries from null to DiskBoundaries{directories=[DataDirectory{location=/home/cjx/Downloads/depart0/depart/data/data}], positions=null, ringVersion=0, directoriesVersion=0} for keyspace_0.globalReplicaTable
01:37:25.890 [org.apache.cassandra.test.microbench.ReadWriteTest.read-jmh-worker-1] DEBUG org.apache.cassandra.config.Schema - Adding org.apache.cassandra.config.CFMetaData#44380385[cfId=bfe68a70-569b-11ed-974a-eb765e7c415c,ksName=keyspace_0,cfName=globalReplicaTable,flags=[COMPOUND],params=TableParams{comment=, read_repair_chance=0.0, dclocal_read_repair_chance=0.1, bloom_filter_fp_chance=0.01, crc_check_chance=1.0, gc_grace_seconds=864000, default_time_to_live=0, memtable_flush_period_in_ms=0, min_index_interval=128, max_index_interval=2048, speculative_retry=99PERCENTILE, caching={'keys' : 'ALL', 'rows_per_partition' : 'NONE'}, compaction=CompactionParams{class=org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy, options={max_threshold=32, min_threshold=4}}, compression=org.apache.cassandra.schema.CompressionParams#920394f7, extensions={}, cdc=false},comparator=comparator(org.apache.cassandra.db.marshal.LongType),partitionColumns=[[] | [commentid]],partitionKeyColumns=[userid],clusteringColumns=[picid],keyValidator=org.apache.cassandra.db.marshal.LongType,columnMetadata=[userid, picid, commentid],droppedColumns={},triggers=[],indexes=[]] to cfIdMap
Writing 50k
<failure>
java.lang.IndexOutOfBoundsException: Index: 0, Size: 0
at java.util.ArrayList.rangeCheck(ArrayList.java:659)
at java.util.ArrayList.get(ArrayList.java:435)
at org.apache.cassandra.locator.TokenMetadata.firstToken(TokenMetadata.java:1079)
at org.apache.cassandra.locator.AbstractReplicationStrategy.getNaturalEndpoints(AbstractReplicationStrategy.java:107)
at org.apache.cassandra.service.StorageService.getNaturalEndpoints(StorageService.java:4066)
at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:619)
at org.apache.cassandra.db.Mutation.apply(Mutation.java:227)
at org.apache.cassandra.db.Mutation.apply(Mutation.java:232)
at org.apache.cassandra.db.Mutation.apply(Mutation.java:241)
at org.apache.cassandra.cql3.statements.ModificationStatement.executeInternalWithoutCondition(ModificationStatement.java:587)
at org.apache.cassandra.cql3.statements.ModificationStatement.executeInternal(ModificationStatement.java:581)
at org.apache.cassandra.cql3.QueryProcessor.executeInternal(QueryProcessor.java:315)
at org.apache.cassandra.cql3.CQLTester.executeFormattedQuery(CQLTester.java:792)
at org.apache.cassandra.cql3.CQLTester.execute(CQLTester.java:780)
at org.apache.cassandra.test.microbench.ReadWriteTest.setup(ReadWriteTest.java:94)
at org.apache.cassandra.test.microbench.generated.ReadWriteTest_read_jmhTest._jmh_tryInit_f_readwritetest0_G(ReadWriteTest_read_jmhTest.java:438)
at org.apache.cassandra.test.microbench.generated.ReadWriteTest_read_jmhTest.read_Throughput(ReadWriteTest_read_jmhTest.java:71)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.openjdk.jmh.runner.BenchmarkHandler$BenchmarkTask.call(BenchmarkHandler.java:453)
at org.openjdk.jmh.runner.BenchmarkHandler$BenchmarkTask.call(BenchmarkHandler.java:437)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
`
then I use ant microbench -Dbenchmark.name=ReadWriteTest to run it, also it leads to the same error.
Any and all help would be appreciated.
Thanks.
of course I run the microbenchs with cassandra running, I also meet mistake when I shut down cassandra
`
INFO [StorageServiceShutdownHook] 2022-10-28 00:58:28,491 Server.java:176 - Stop listening for CQL clients
INFO [StorageServiceShutdownHook] 2022-10-28 00:58:28,491 Gossiper.java:1551 - Announcing shutdown
INFO [StorageServiceShutdownHook] 2022-10-28 00:58:28,492 StorageService.java:2454 - Node /192.168.199.135 state jump to shutdown
INFO [StorageServiceShutdownHook] 2022-10-28 00:58:30,495 MessagingService.java:981 - Waiting for messaging service to quiesce
INFO [ACCEPT-/192.168.199.135] 2022-10-28 00:58:30,495 MessagingService.java:1336 - MessagingService has terminated the accept() thread
WARN [MemtableFlushWriter:3] 2022-10-28 00:58:30,498 NativeLibrary.java:304 - open(/home/cjx/Downloads/depart0/depart/data/data/system_auth/roles-5bc52802de2535edaeab188eecebb090, O_RDONLY) failed, errno (2).
ERROR [MemtableFlushWriter:3] 2022-10-28 00:58:30,500 LogTransaction.java:272 - Transaction log [md_txn_flush_4ff62d10-5696-11ed-80e2-25f08a3965a3.log in /home/cjx/Downloads/depart0/depart/data/data/system_auth/roles-5bc52802de2535edaeab188eecebb090] indicates txn was not completed, trying to abort it now
ERROR [MemtableFlushWriter:3] 2022-10-28 00:58:30,505 LogTransaction.java:275 - Failed to abort transaction log [md_txn_flush_4ff62d10-5696-11ed-80e2-25f08a3965a3.log in /home/cjx/Downloads/depart0/depart/data/data/system_auth/roles-5bc52802de2535edaeab188eecebb090]
java.lang.RuntimeException: java.nio.file.NoSuchFileException: /home/cjx/Downloads/depart0/depart/data/data/system_auth/roles-5bc52802de2535edaeab188eecebb090/md_txn_flush_4ff62d10-5696-11ed-80e2-25f08a3965a3.log
at org.apache.cassandra.io.util.FileUtils.write(FileUtils.java:590) ~[main/:na]
at org.apache.cassandra.io.util.FileUtils.appendAndSync(FileUtils.java:571) ~[main/:na]
at org.apache.cassandra.db.lifecycle.LogReplica.append(LogReplica.java:85) ~[main/:na]
at org.apache.cassandra.db.lifecycle.LogReplicaSet.lambda$null$5(LogReplicaSet.java:210) ~[main/:na]
at org.apache.cassandra.utils.Throwables.perform(Throwables.java:113) ~[main/:na]
at org.apache.cassandra.utils.Throwables.perform(Throwables.java:103) ~[main/:na]
at org.apache.cassandra.db.lifecycle.LogReplicaSet.append(LogReplicaSet.java:210) ~[main/:na]
at org.apache.cassandra.db.lifecycle.LogFile.addRecord(LogFile.java:338) ~[main/:na]
at org.apache.cassandra.db.lifecycle.LogFile.abort(LogFile.java:255) ~[main/:na]
at org.apache.cassandra.utils.Throwables.perform(Throwables.java:113) ~[main/:na]
at org.apache.cassandra.utils.Throwables.perform(Throwables.java:103) ~[main/:na]
at org.apache.cassandra.utils.Throwables.perform(Throwables.java:98) ~[main/:na]
at org.apache.cassandra.db.lifecycle.LogTransaction$TransactionTidier.run(LogTransaction.java:273) [main/:na]
at org.apache.cassandra.db.lifecycle.LogTransaction$TransactionTidier.tidy(LogTransaction.java:257) [main/:na]
at org.apache.cassandra.utils.concurrent.Ref$GlobalState.release(Ref.java:322) [main/:na]
at org.apache.cassandra.utils.concurrent.Ref$State.ensureReleased(Ref.java:200) [main/:na]
at org.apache.cassandra.utils.concurrent.Ref.ensureReleased(Ref.java:120) [main/:na]
at org.apache.cassandra.db.lifecycle.LogTransaction.complete(LogTransaction.java:392) [main/:na]
at org.apache.cassandra.db.lifecycle.LogTransaction.doAbort(LogTransaction.java:409) [main/:na]
at org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.abort(Transactional.java:144) [main/:na]
at org.apache.cassandra.db.lifecycle.LifecycleTransaction.doAbort(LifecycleTransaction.java:244) [main/:na]
at org.apache.cassandra.utils.concurrent.Transactional$AbstractTransactional.abort(Transactional.java:144) [main/:na]
at org.apache.cassandra.db.ColumnFamilyStore$Flush.flushMemtable(ColumnFamilyStore.java:1234) [main/:na]
at org.apache.cassandra.db.ColumnFamilyStore$Flush.run(ColumnFamilyStore.java:1180) [main/:na]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_332]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_332]
at org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:81) [main/:na]
at java.lang.Thread.run(Thread.java:750) ~[na:1.8.0_332]
Caused by: java.nio.file.NoSuchFileException: /home/cjx/Downloads/depart0/depart/data/data/system_auth/roles-5bc52802de2535edaeab188eecebb090/md_txn_flush_4ff62d10-5696-11ed-80e2-25f08a3965a3.log
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86) ~[na:1.8.0_332]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) ~[na:1.8.0_332]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) ~[na:1.8.0_332]
at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) ~[na:1.8.0_332]
at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) ~[na:1.8.0_332]
at java.nio.file.Files.newOutputStream(Files.java:216) ~[na:1.8.0_332]
at java.nio.file.Files.write(Files.java:3351) ~[na:1.8.0_332]
at org.apache.cassandra.io.util.FileUtils.write(FileUtils.java:583) ~[main/:na]
... 27 common frames omitted
`
it's so wired and I could not know how to fix it
Related
java.io.IOException: Filesystem closed
my spark job; val df = spark.sql(s"""""") df.write.mode("append").json("hdfs://xxx-nn-ha/user/b_me/df") Error: 2019-01-31 19:56:36 INFO CoarseGrainedExecutorBackend:54 - Driver commanded a shutdown 2019-01-31 19:56:36 INFO MemoryStore:54 - MemoryStore cleared 2019-01-31 19:56:36 INFO BlockManager:54 - BlockManager stopped 2019-01-31 19:56:36 INFO ShutdownHookManager:54 - Shutdown hook called 2019-01-31 19:56:36 ERROR Executor:91 - Exception in task 4120.0 in stage 5.0 (TID 5402) java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:808) at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:868) at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:934) at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:735) at java.io.DataInputStream.readInt(DataInputStream.java:387) at org.apache.hadoop.io.SequenceFile$Reader.readBlock(SequenceFile.java:2178) at org.apache.hadoop.io.SequenceFile$Reader.next(SequenceFile.java:2585) at org.apache.hadoop.mapred.SequenceFileRecordReader.next(SequenceFileRecordReader.java:82) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:277) at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:214) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage12.processNext(Unknown Source) at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$10$$anon$1.hasNext(WholeStageCodegenExec.scala:614) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:125) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) at org.apache.spark.scheduler.Task.run(Task.scala:109) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:369) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2019-01-31 19:56:36 INFO Executor:54 - Not reporting error to driver during JVM shutdown. End of LogType:stdout I got the java.io.IOException: Filesystem closed error. Why? I have no hints. Any ideas welcomed. Thanks UPDATE There is a WARN: 2019-02-01 12:01:39 INFO YarnAllocator:54 - Driver requested a total number of 2007 executor(s). 2019-02-01 12:01:39 INFO ExecutorAllocationManager:54 - Requesting 968 new executors because tasks are backlogged (new desired total will be 2007) 2019-02-01 12:01:39 INFO ExecutorAllocationManager:54 - New executor 25 has registered (new total is 26) 2019-02-01 12:01:39 WARN ApplicationMaster:87 - Reporter thread fails 1 time(s) in a row. org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException: Too many containers asked, 1365198 at org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils.normalizeAndValidateRequests(RMServerUtils.java:128) at org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:511) at org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60) at org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:616) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:969) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2206) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2202) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1714) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2202) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53) at org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:101) at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.allocate(ApplicationMasterProtocolPBClientImpl.java:79) at sun.reflect.GeneratedMethodAccessor36.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy22.allocate(Unknown Source) at org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl.allocate(AMRMClientImpl.java:277) at org.apache.spark.deploy.yarn.YarnAllocator.allocateResources(YarnAllocator.scala:268) at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$3.run(ApplicationMaster.scala:556)
If you look closer at DFSClient.checkOpen code you will see the following: void checkOpen() throws IOException { if (!clientRunning) { IOException result = new IOException("Filesystem closed"); throw result; } } Let's find all accessors of clientRunning field. Only close method really changes it. Lets take a look at it: #Override public synchronized void close() throws IOException { try { if(clientRunning) { closeAllFilesBeingWritten(false); clientRunning = false; getLeaseRenewer().closeClient(this); // close connections to the namenode closeConnectionToNamenode(); } } finally { if (provider != null) { provider.close(); } } } So the main problem in your job is that it tries to write the value, although the FS is already closed. Make sure you don't close your FS before you do any job. You can also increase the logging level to find the cause.
Spring boot integration mail - authorization failed error
Following is my spring configuration bean for pop3 connection, #Bean public IntegrationFlow mailListener() { return IntegrationFlows.from(Mail.pop3InboundAdapter("pop3://sample.test:Sample2test_1#xxx.xx.x.xx/INBOX") .shouldDeleteMessages(true).get(), e -> e.poller(Pollers.fixedRate(5000).maxMessagesPerPoll(1))) .<Message>handle((payload, header) -> logMail(payload)) .get(); } Also double checked the credentials that looks good. Getting the below exception, 2018-06-22 19:27:54.351 ERROR 2092 --- [ask-scheduler-2] o.s.integration.handler.LoggingHandler : org.springframework.messaging.MessagingException: failure occurred while polling for mail; nested exception is javax.mail.AuthenticationFailedException: authorization failed at org.springframework.integration.mail.MailReceivingMessageSource.receive(MailReceivingMessageSource.java:131) at org.springframework.integration.endpoint.SourcePollingChannelAdapter.receiveMessage(SourcePollingChannelAdapter.java:224) at org.springframework.integration.endpoint.AbstractPollingEndpoint.doPoll(AbstractPollingEndpoint.java:245) at org.springframework.integration.endpoint.AbstractPollingEndpoint.access$000(AbstractPollingEndpoint.java:58) at org.springframework.integration.endpoint.AbstractPollingEndpoint$1.call(AbstractPollingEndpoint.java:190) at org.springframework.integration.endpoint.AbstractPollingEndpoint$1.call(AbstractPollingEndpoint.java:186) at org.springframework.integration.endpoint.AbstractPollingEndpoint$Poller$1.run(AbstractPollingEndpoint.java:353) at org.springframework.integration.util.ErrorHandlingTaskExecutor$1.run(ErrorHandlingTaskExecutor.java:55) at org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:50) at org.springframework.integration.util.ErrorHandlingTaskExecutor.execute(ErrorHandlingTaskExecutor.java:51) at org.springframework.integration.endpoint.AbstractPollingEndpoint$Poller.run(AbstractPollingEndpoint.java:344) at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81) at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.util.concurrent.FutureTask.run(Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Caused by: javax.mail.AuthenticationFailedException: authorization failed at com.sun.mail.pop3.POP3Store.protocolConnect(POP3Store.java:208) at javax.mail.Service.connect(Service.java:295) at javax.mail.Service.connect(Service.java:176) at javax.mail.Service.connect(Service.java:125) at org.springframework.integration.mail.AbstractMailReceiver.connectStoreIfNecessary(AbstractMailReceiver.java:286) at org.springframework.integration.mail.AbstractMailReceiver.openFolder(AbstractMailReceiver.java:293) at org.springframework.integration.mail.AbstractMailReceiver.receive(AbstractMailReceiver.java:319) at org.springframework.integration.mail.MailReceivingMessageSource.receive(MailReceivingMessageSource.java:112) ... 19 more
There is an option like this: /** * The Java Mail properties. * #param javaMailProperties the javaMailProperties. * #return the spec. * #see AbstractMailReceiver#setJavaMailProperties(Properties) */ public S javaMailProperties(Properties javaMailProperties) { You can specify there mail.debug = true to trace what's going on with mail protocol during interaction. UPDATE We have this in your logs: DEBUG POP3: connecting to host "172.16.1.45", port 995, isSSL false 19:54:46.746 [task-scheduler-1] DEBUG org.springframework.beans.factory.support.DefaultListableBeanFactory - Returning cached instance of singleton bean 'errorChannel' 19:54:46.753 [task-scheduler-1] DEBUG org.springframework.integration.channel.PublishSubscribeChannel - preSend on channel 'errorChannel', message: ErrorMessage [payload=org.springframework.messaging.MessagingException: failure occurred while polling for mail; nested exception is javax.mail.MessagingException: Connect failed; nested exception is: java.net.ConnectException: Connection refused: connect, headers={id=290f4c5a-2dfe-d7cb-0637-a0e9f0394daa, timestamp=1529677486753}] So, it can't just connect to the 172.16.1.45:995. There is no authentication handshake yet.
Scala to HBASE- ZooKeeperRegistry: ClusterId read in ZooKeeper is null
I am using the following code to connect to my HBASE which is running on local sandbox. I am using the following code, zookeeper and hbase master are running finw. I can scan the HBASE table by going to the HBASE Shell. Not sure whats going wrong here?: object HBaseWrite { def main(args: Array[String]) { System.setProperty("hadoop.home.dir", "c://hadoop"); val sparkConf = new SparkConf().setAppName("HBaseWrite").setMaster("local[*]").set("spark.driver.allowMultipleContexts","true").set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") val sc = new SparkContext(sparkConf) val conf = HBaseConfiguration.create() val outputTable = "TABLE1" System.setProperty("user.name", "hdfs") System.setProperty("HADOOP_USER_NAME", "hdfs") conf.set("hbase.master", "localhost:60000") conf.setInt("timeout", 120000) conf.set("hbase.zookeeper.quorum", "192.168.163.129/sandbox.hortonworks.com") conf.set("zookeeper.znode.parent", "/hbase-unsecure") conf.setInt("hbase.client.scanner.caching", 10000) sparkConf.registerKryoClasses(Array(classOf[org.apache.hadoop.hbase.client.Result])) val jobConfig: JobConf = new JobConf(conf,this.getClass) jobConfig.setOutputFormat(classOf[TableOutputFormat]) jobConfig.set(TableOutputFormat.OUTPUT_TABLE,outputTable) val x = 12 val y = 15 val z = 25 var newarray = Array(x,y,z) val newrddtohbase = sc.parallelize(newarray) val convertFunc = convert _ new PairRDDFunctions(newrddtohbase.map(convertFunc)).saveAsHadoopDataset(jobConfig) sc.stop() } def convert(a:Int) : Tuple2[ImmutableBytesWritable,Put] = { val p = new Put(Bytes.toBytes(a)) p.add(Bytes.toBytes("columnfamily"), Bytes.toBytes("col_1"), Bytes.toBytes(a)) new Tuple2[ImmutableBytesWritable,Put](new ImmutableBytesWritable(a.toString.getBytes()), p); } } and get the following error: ZooKeeperRegistry: ClusterId read in ZooKeeper is null 17/03/21 20:13:37 INFO ZooKeeperRegistry: ClusterId read in ZooKeeper is null 17/03/21 20:13:37 INFO ZooKeeperRegistry: ClusterId read in ZooKeeper is null 17/03/21 20:13:37 INFO ZooKeeperRegistry: ClusterId read in ZooKeeper is null 17/03/21 20:13:37 INFO SparkHadoopMapRedUtil: No need to commit output of task because needsTaskCommit=false: attempt_201703212013_0000_m_000000_0 17/03/21 20:13:37 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 874 bytes result sent to driver 17/03/21 20:13:37 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 816 ms on localhost (1/4) 17/03/21 20:13:37 ERROR Executor: Exception in task 3.0 in stage 0.0 (TID 3) java.lang.RuntimeException: java.lang.NullPointerException at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:208) at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.loadCache(ClientSmallReversedScanner.java:211) at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.next(ClientSmallReversedScanner.java:185) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegionInMeta(ConnectionManager.java:1249) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1155) at org.apache.hadoop.hbase.client.AsyncProcess.submit(AsyncProcess.java:370) at org.apache.hadoop.hbase.client.AsyncProcess.submit(AsyncProcess.java:321) at org.apache.hadoop.hbase.client.BufferedMutatorImpl.backgroundFlushCommits(BufferedMutatorImpl.java:206) at org.apache.hadoop.hbase.client.BufferedMutatorImpl.close(BufferedMutatorImpl.java:158) at org.apache.hadoop.hbase.mapred.TableOutputFormat$TableRecordWriter.close(TableOutputFormat.java:66) at org.apache.spark.SparkHadoopWriter.close(SparkHadoopWriter.scala:102) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$8.apply$mcV$sp(PairRDDFunctions.scala:1211) at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1366) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1211) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1190) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:70) at org.apache.spark.scheduler.Task.run(Task.scala:86) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Caused by: java.lang.NullPointerException at org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.getMetaReplicaNodes(ZooKeeperWatcher.java:395) at org.apache.hadoop.hbase.zookeeper.MetaTableLocator.blockUntilAvailable(MetaTableLocator.java:553) at org.apache.hadoop.hbase.client.ZooKeeperRegistry.getMetaRegionLocation(ZooKeeperRegistry.java:61) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateMeta(ConnectionManager.java:1185) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1152) at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:300) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:151) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:59) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) ... 20 more 17/03/21 20:13:37 ERROR Executor: Exception in task 2.0 in stage 0.0 (TID 2) java.lang.RuntimeException: java.lang.NullPointerException at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:208) at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.loadCache(ClientSmallReversedScanner.java:211) at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.next(ClientSmallReversedScanner.java:185) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegionInMeta(ConnectionManager.java:1249) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1155) at org.apache.hadoop.hbase.client.AsyncProcess.submit(AsyncProcess.java:370) at org.apache.hadoop.hbase.client.AsyncProcess.submit(AsyncProcess.java:321) at org.apache.hadoop.hbase.client.BufferedMutatorImpl.backgroundFlushCommits(BufferedMutatorImpl.java:206) at org.apache.hadoop.hbase.client.BufferedMutatorImpl.close(BufferedMutatorImpl.java:158) at org.apache.hadoop.hbase.mapred.TableOutputFormat$TableRecordWriter.close(TableOutputFormat.java:66) at org.apache.spark.SparkHadoopWriter.close(SparkHadoopWriter.scala:102) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$8.apply$mcV$sp(PairRDDFunctions.scala:1211) at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1366) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1211) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1190) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:70) at org.apache.spark.scheduler.Task.run(Task.scala:86) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Caused by: java.lang.NullPointerException at org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.getMetaReplicaNodes(ZooKeeperWatcher.java:395) at org.apache.hadoop.hbase.zookeeper.MetaTableLocator.blockUntilAvailable(MetaTableLocator.java:553) at org.apache.hadoop.hbase.client.ZooKeeperRegistry.getMetaRegionLocation(ZooKeeperRegistry.java:61) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateMeta(ConnectionManager.java:1185) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1152) at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:300) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:151) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:59) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) ... 20 more 17/03/21 20:13:37 ERROR Executor: Exception in task 1.0 in stage 0.0 (TID 1) java.lang.RuntimeException: java.lang.NullPointerException at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:208) at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.loadCache(ClientSmallReversedScanner.java:211) at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.next(ClientSmallReversedScanner.java:185) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegionInMeta(ConnectionManager.java:1249) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1155) at org.apache.hadoop.hbase.client.AsyncProcess.submit(AsyncProcess.java:370) at org.apache.hadoop.hbase.client.AsyncProcess.submit(AsyncProcess.java:321) at org.apache.hadoop.hbase.client.BufferedMutatorImpl.backgroundFlushCommits(BufferedMutatorImpl.java:206) at org.apache.hadoop.hbase.client.BufferedMutatorImpl.close(BufferedMutatorImpl.java:158) at org.apache.hadoop.hbase.mapred.TableOutputFormat$TableRecordWriter.close(TableOutputFormat.java:66) at org.apache.spark.SparkHadoopWriter.close(SparkHadoopWriter.scala:102) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$8.apply$mcV$sp(PairRDDFunctions.scala:1211) at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1366) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1211) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1190) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:70) at org.apache.spark.scheduler.Task.run(Task.scala:86) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Caused by: java.lang.NullPointerException at org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.getMetaReplicaNodes(ZooKeeperWatcher.java:395) at org.apache.hadoop.hbase.zookeeper.MetaTableLocator.blockUntilAvailable(MetaTableLocator.java:553) at org.apache.hadoop.hbase.client.ZooKeeperRegistry.getMetaRegionLocation(ZooKeeperRegistry.java:61) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateMeta(ConnectionManager.java:1185) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1152) at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:300) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:151) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:59) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) ... 20 more 17/03/21 20:13:37 WARN TaskSetManager: Lost task 2.0 in stage 0.0 (TID 2, localhost): java.lang.RuntimeException: java.lang.NullPointerException at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:208) at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.loadCache(ClientSmallReversedScanner.java:211) at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.next(ClientSmallReversedScanner.java:185) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegionInMeta(ConnectionManager.java:1249) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1155) at org.apache.hadoop.hbase.client.AsyncProcess.submit(AsyncProcess.java:370) at org.apache.hadoop.hbase.client.AsyncProcess.submit(AsyncProcess.java:321) at org.apache.hadoop.hbase.client.BufferedMutatorImpl.backgroundFlushCommits(BufferedMutatorImpl.java:206) at org.apache.hadoop.hbase.client.BufferedMutatorImpl.close(BufferedMutatorImpl.java:158) at org.apache.hadoop.hbase.mapred.TableOutputFormat$TableRecordWriter.close(TableOutputFormat.java:66) at org.apache.spark.SparkHadoopWriter.close(SparkHadoopWriter.scala:102) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13$$anonfun$apply$8.apply$mcV$sp(PairRDDFunctions.scala:1211) at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1366) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1211) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1$$anonfun$13.apply(PairRDDFunctions.scala:1190) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:70) at org.apache.spark.scheduler.Task.run(Task.scala:86) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Caused by: java.lang.NullPointerException at org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.getMetaReplicaNodes(ZooKeeperWatcher.java:395) at org.apache.hadoop.hbase.zookeeper.MetaTableLocator.blockUntilAvailable(MetaTableLocator.java:553) at org.apache.hadoop.hbase.client.ZooKeeperRegistry.getMetaRegionLocation(ZooKeeperRegistry.java:61) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateMeta(ConnectionManager.java:1185) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1152) at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:300) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:151) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:59) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) ... 20 more Please help, I am stuck on this bit. HBASE Master and Zookeeper are running fine.
Exception while executing Spark job in cluster mode
I have a simple spark job as below SparkConf conf = new SparkConf(); conf.setMaster("spark://v-in-spark-01.synygy.net:7077"); SparkContext sc = new SparkContext(conf); JavaSparkContext context = new JavaSparkContext(sc); List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> distData = context.parallelize(data); distData.foreach(MainClass::printValue); sc.stop(); and a launcher app Process spark = new SparkLauncher() .setAppName("simplesparkjob") .setAppResource("file:///home/spark/programs/simplesparkjob-all.jar") .setMainClass("com.spmsoftware.main.MainClass") .setMaster("SPARK_MASTER:6066") .setSparkHome("SPARK_HOME") .setVerbose(true) .setDeployMode("cluster") .launch(); After executing launcher the job gets submitted properly and I can see it on mster UI but I can see the output in stdout. Instead I am getting following exception in stderr : Exception in thread "main" java.lang.reflect.UndeclaredThrowableException at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.spark.deploy.SparkHadoopUtil.runAsSparkUser(SparkHadoopUtil.scala:70) at org.apache.spark.executor.CoarseGrainedExecutorBackend$.run(CoarseGrainedExecutorBackend.scala:174) at org.apache.spark.executor.CoarseGrainedExecutorBackend$.main(CoarseGrainedExecutorBackend.scala:270) at org.apache.spark.executor.CoarseGrainedExecutorBackend.main(CoarseGrainedExecutorBackend.scala) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult at org.apache.spark.rpc.RpcTimeout$$anonfun$1.applyOrElse(RpcTimeout.scala:77) at org.apache.spark.rpc.RpcTimeout$$anonfun$1.applyOrElse(RpcTimeout.scala:75) at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36) at org.apache.spark.rpc.RpcTimeout$$anonfun$addMessageIfTimeout$1.applyOrElse(RpcTimeout.scala:59) at org.apache.spark.rpc.RpcTimeout$$anonfun$addMessageIfTimeout$1.applyOrElse(RpcTimeout.scala:59) at scala.PartialFunction$OrElse.apply(PartialFunction.scala:167) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:83) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:88) at org.apache.spark.executor.CoarseGrainedExecutorBackend$$anonfun$run$1.apply$mcV$sp(CoarseGrainedExecutorBackend.scala:188) at org.apache.spark.deploy.SparkHadoopUtil$$anon$1.run(SparkHadoopUtil.scala:71) at org.apache.spark.deploy.SparkHadoopUtil$$anon$1.run(SparkHadoopUtil.scala:70) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1656) ... 4 more Caused by: java.io.IOException: Failed to connect to /127.0.0.1:42460 at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:228) at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:179) at org.apache.spark.rpc.netty.NettyRpcEnv.createClient(NettyRpcEnv.scala:197) at org.apache.spark.rpc.netty.Outbox$$anon$1.call(Outbox.scala:191) at org.apache.spark.rpc.netty.Outbox$$anon$1.call(Outbox.scala:187) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.net.ConnectException: Connection refused: /127.0.0.1:42460 at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717) at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:224) at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:289) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:528) at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:468) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:382) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:354) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111) ... 1 more What this error is? what is it that I am missing?
How do I reconnect to Cassandra using Hector?
I have the following code: StringSerializer ss = StringSerializer.get(); String cf = "TEST"; CassandraHostConfigurator conf = new CassandraHostConfigurator("localhost:9160"); conf.setCassandraThriftSocketTimeout(40000); conf.setExhaustedPolicy(ExhaustedPolicy.WHEN_EXHAUSTED_BLOCK); conf.setRetryDownedHostsDelayInSeconds(5); conf.setRetryDownedHostsQueueSize(128); conf.setRetryDownedHosts(true); conf.setLoadBalancingPolicy(new LeastActiveBalancingPolicy()); String key = Long.toString(System.currentTimeMillis()); Cluster cluster = HFactory.getOrCreateCluster("TestCluster", conf); Keyspace keyspace = HFactory.createKeyspace("TestCluster", cluster); Mutator<String> mutator = HFactory.createMutator(keyspace, StringSerializer.get()); int count = 0; while (!"q".equals(new Scanner( System.in).next())) { try{ mutator.insert(key, cf, HFactory.createColumn("column_" + count, "v_" + count, ss, ss)); count++; } catch (Exception e) { e.printStackTrace(); } } and I can write some values using it, but when I restart cassandra, it fails. Here is the log: [15:11:07] INFO [CassandraHostRetryService ] Downed Host Retry service started with >queue size 128 and retry delay 5s [15:11:07] INFO [JmxMonitor ] Registering JMX >me.prettyprint.cassandra.service_ASG:ServiceType=hector,MonitorType=hector [15:11:17] ERROR [HThriftClient ] Could not flush transport (to be expected >if the pool is shutting down) in close for client: CassandraClient org.apache.thrift.transport.TTransportException: java.net.SocketException: Broken pipe at >org.apache.thrift.transport.TIOStreamTransport.write(TIOStreamTransport.java:147) at org.apache.thrift.transport.TFramedTransport.flush(TFramedTransport.java:156) at >me.prettyprint.cassandra.connection.client.HThriftClient.close(HThriftClient.java:98) at >me.prettyprint.cassandra.connection.client.HThriftClient.close(HThriftClient.java:26) at >me.prettyprint.cassandra.connection.HConnectionManager.closeClient(HConnectionManager.java:308) at >me.prettyprint.cassandra.connection.HConnectionManager.operateWithFailover(HConnectionManager.java:257) at >me.prettyprint.cassandra.model.ExecutingKeyspace.doExecuteOperation(ExecutingKeyspace.java:97) at me.prettyprint.cassandra.model.MutatorImpl.execute(MutatorImpl.java:243) at me.prettyprint.cassandra.model.MutatorImpl.insert(MutatorImpl.java:69) at com.app.App.main(App.java:40) Caused by: java.net.SocketException: Broken pipe at java.net.SocketOutputStream.socketWrite0(Native Method) at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:109) at java.net.SocketOutputStream.write(SocketOutputStream.java:153) at >org.apache.thrift.transport.TIOStreamTransport.write(TIOStreamTransport.java:145) ... 9 more [15:11:17] ERROR [HConnectionManager ] MARK HOST AS DOWN TRIGGERED for host >localhost(127.0.0.1):9160 [15:11:17] ERROR [HConnectionManager ] Pool state on shutdown: >:{localhost(127.0.0.1):9160}; IsActive?: true; Active: 1; Blocked: 0; Idle: 15; NumBeforeExhausted: 49 [15:11:17] INFO [ConcurrentHClientPool ] Shutdown triggered on :{localhost(127.0.0.1):9160} [15:11:17] INFO [ConcurrentHClientPool ] Shutdown complete on :{localhost(127.0.0.1):9160} [15:11:17] INFO [CassandraHostRetryService ] Host detected as down was added to retry queue: localhost(127.0.0.1):9160 [15:11:17] WARN [HConnectionManager ] Could not fullfill request on this host CassandraClient [15:11:17] WARN [HConnectionManager ] Exception: me.prettyprint.hector.api.exceptions.HectorTransportException: org.apache.thrift.transport.TTransportException: java.net.SocketException: Broken pipe at >me.prettyprint.cassandra.connection.client.HThriftClient.getCassandra(HThriftClient.java:82) at >me.prettyprint.cassandra.connection.HConnectionManager.operateWithFailover(HConnectionManager.java:236) at >me.prettyprint.cassandra.model.ExecutingKeyspace.doExecuteOperation(ExecutingKeyspace.java:97) at me.prettyprint.cassandra.model.MutatorImpl.execute(MutatorImpl.java:243) at me.prettyprint.cassandra.model.MutatorImpl.insert(MutatorImpl.java:69) at com.app.App.main(App.java:40) Caused by: org.apache.thrift.transport.TTransportException: java.net.SocketException: Broken pipe at org.apache.thrift.transport.TIOStreamTransport.write(TIOStreamTransport.java:147) at org.apache.thrift.transport.TFramedTransport.flush(TFramedTransport.java:157) at org.apache.cassandra.thrift.Cassandra$Client.send_set_keyspace(Cassandra.java:466) at org.apache.cassandra.thrift.Cassandra$Client.set_keyspace(Cassandra.java:455) at >me.prettyprint.cassandra.connection.client.HThriftClient.getCassandra(HThriftClient.java:78) ... 5 more Caused by: java.net.SocketException: Broken pipe at java.net.SocketOutputStream.socketWrite0(Native Method) at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:109) at java.net.SocketOutputStream.write(SocketOutputStream.java:153) at >org.apache.thrift.transport.TIOStreamTransport.write(TIOStreamTransport.java:145) ... 9 more [15:11:17] INFO [HConnectionManager ] Client CassandraClient released to inactive or dead pool. Closing. [15:11:17] INFO [HConnectionManager ] Client CassandraClient released to inactive or dead pool. Closing. [15:11:17] INFO [HConnectionManager ] Added host localhost(127.0.0.1):9160 to pool
You have set - conf.setRetryDownedHostsDelayInSeconds(5); Try to to wait after the restart for more than 5 seconds. Also, you may need to upgrade. What is the size thrift_max_message_length_in_mb you have set? Kind regards.