How to retrieve more than 100 messages from the history of a PubNub channel? - history

The page about the PubNub History API states that
The history() function returns a list of up to 100 messages, the start
time token and the ending time token.
Is there a way to retrieve more than the 100 messages?
I'm currently not a paying customer of PubNub.

PubNub Load History More than 100 Messages
Sometimes you want to slice back in time over a linear stream of data. And often you'll want to do this at different levels of granularity. That is why PubNub Storage and Playback APIs provide maximum level of flexibility. However sometimes it ends up being a bit tricky to load data with the preferred result set.
PubNub Real-Time Network Storage and Playback
There are several considerations you may be seeking when loading transaction history over timelines that can potentially span millions of message in the transaction set. There are some great options available to you and we will cover two of them right now. The examples will be coded in JavaScript. The first example loads a summary of the data by grabbing the snapshots for the beginning of each hour for the past 24 hours. The second example shows you how to load all transactions in full detail and maximum granularity.
All Reference Files can be found on this GIST: Loading History from PubNub Mt.Gox Trades
Example PubNub Mt.Gox History JavaScript Usage
<script src="https://cdn.pubnub.com/pubnub.min.js"></script>
<script src="mtgox-history.js"></script>
<script>(function(){
// LOAD HOURLY SUMMARY
MTGOX.history.hourly({
channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f',
data : function(response) { console.log(JSON.stringify(response)) },
error : function() { console.log("NETWORK ERROR") }
});
// LOAD ALL WITH LIMITER OPTION
MTGOX.history.full({
limit : 500, // SET LIMIT AS HIGH AS NEEDED TO LOAD MORE!
channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f',
data : function(messages) { console.log(messages) },
error : function(e) { console.log("NETWORK ERROR") }
});
})();</script>
NOTE: Running MTGOX.history.hourly() method will generate a list of snapshots per hour over the last 24 hours.
NOTE: Running MTGOX.history.full() method will generate maximum resolution detail with a lot of data. You can get a full dump or partial dump as needed; and you should increase the limit parameter in order to grab more data points.
This following JavaScript file will provide you the MTGOX interface.
PubNub Mt.Gox History JavaScript Loader
//
// mtgox-history.js
//
(function(){
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// INITIALIZE PUBNUB
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
var pubnub = PUBNUB.init({
subscribe_key : 'sub-c-50d56e1e-2fd9-11e3-a041-02ee2ddab7fe'
});
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// MTGOX HISTORY INTERFACE
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
window.MTGOX = {
history : {
hourly : hourly,
full : full
}
};
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// GET ALL DATA FOREVER (WITH LIMIT OF COURSE)
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
/*
MTGOX.history.full({
limit : 1000,
channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f',
data : function(messages) { console.log(messages) },
error : function(e) { console.log("NETWORK ERROR") }
});
*/
function full(args) {
var chan = args['channel'] ||'d5f06780-30a8-4a48-a2f8-7ed181b4a13f'
, callback = args['data'] || function(){}
, error = args['error'] || function(){}
, limit = +args['limit'] || 5000
, start = 0
, count = 100
, history = []
, params = {
channel : chan,
count : count,
callback : function(messages) {
var msgs = messages[0];
start = messages[1];
params.start = start;
PUBNUB.each( msgs.reverse(), function(m) {history.push(m)} );
if (history.length >= limit) return callback(history);
if (msgs.length < count) return callback(history);
count = 100;
add_messages();
},
error : function(e) {
callback(history);
error(history);
}
};
add_messages();
function add_messages() { pubnub.history(params) }
}
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// GET 24 HOURS IN HOURLY INCREMENTS
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
/*
MTGOX.history.hourly({
channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f',
data : function(response) { console.log(response) },
error : function() { console.log('ERROR') }
});
*/
function hourly(setup) {
var limit = 24;
var count = 0;
var chan = setup['channel'] ||'d5f06780-30a8-4a48-a2f8-7ed181b4a13f';
var cb = setup['data'] || function(){};
var eb = setup['error'] || function(){};
var now = new Date();
now.setUTCHours(0);
now.setUTCMinutes(0);
now.setUTCSeconds(0);
now.setUTCMilliseconds(0);
var utc_now = now.getTime();
var vectors = [];
PUBNUB.each( (new Array(limit)).join(',').split(','), function( _, d ) {
var day = utc_now - 3600000 * d;
pubnub.history({
limit : 1,
channel : chan,
start : day * 10000,
error : function() { count++; eb(); },
callback : function(messages) {
// DONE?
if (++count == limit) return cb(vectors);
// ADD TIME SLICES
var res = +(((messages[0][0]||{}).ticker||{}).avg||{}).value;
res && vectors.push([ new Date(day).getUTCHours(), res ]);
// KEEP IT SORTED
vectors.sort(function(a,b){ return a[0] > b[0] && -1 || 1 });
}
})
} );
}
})();
Mt.Gox PubNub Channel Listing for Tickers, Depth and Trades
The following is a list of channels provided by Mt.Gox data feed options you can use in the history channel parameter field.
{
"TICKER.ltcgbp": "0102A446-E4D4-4082-8E83-CC02822F9172",
"TICKER.ltccny": "0290378C-E3D7-4836-8CB1-2BFAE20CC492",
"DEPTH.btchkd": "049F65DC-3AF3-4FFD-85A5-AAC102B2A579",
"DEPTH.btceur": "057BDC6B-9F9C-44E4-BC1A-363E4443CE87",
"TICKER.nmcaud": "08C65460-CBD9-492E-8473-8507DFA66AE6",
"TICKER.btceur": "0BB6DA8B-F6C6-4ECF-8F0D-A544AD948C15",
"DEPTH.btckrw": "0C84BDA7-E613-4B19-AE2A-6D26412C9F70",
"DEPTH.btccny": "0D1ECAD8-E20F-459E-8BED-0BDCF927820F",
"TICKER.btccad": "10720792-084D-45BA-92E3-CF44D9477775",
"DEPTH.btcchf": "113FEC5F-294D-4929-86EB-8CA4C3FD1BED",
"TICKER.ltcnok": "13616AE8-9268-4A43-BDF7-6B8D1AC814A2",
"TICKER.ltcusd": "1366A9F3-92EB-4C6C-9CCC-492A959ECA94",
"TICKER.btcbtc": "13EDFF67-CFA0-4D99-AA76-52BD15D6A058",
"TICKER.ltccad": "18B55737-3F5C-4583-AF63-6EB3951EAD72",
"TICKER.nmccny": "249FDEFD-C6EB-4802-9F54-064BC83908AA",
"DEPTH.btcusd": "24E67E0D-1CAD-4CC0-9E7A-F8523EF460FE",
"TICKER.btcchf": "2644C164-3DB7-4475-8B45-C7042EFE3413",
"DEPTH.btcaud": "296EE352-DD5D-46F3-9BEA-5E39DEDE2005",
"TICKER.btcczk": "2A968B7F-6638-40BA-95E7-7284B3196D52",
"TICKER.btcsgd": "2CB73ED1-07F4-45E0-8918-BCBFDA658912",
"TICKER.nmcjpy": "314E2B7A-A9FA-4249-BC46-B7F662ECBC3A",
"TICKER.btcnmc": "36189B8C-CFFA-40D2-B205-FB71420387AE",
"DEPTH.btcinr": "414FDB18-8F70-471C-A9DF-B3C2740727EA",
"DEPTH.btcsgd": "41E5C243-3D44-4FAD-B690-F39E1DBB86A8",
"TICKER.btcltc": "48B6886F-49C0-4614-B647-BA5369B449A9",
"TICKER.ltceur": "491BC9BB-7CD8-4719-A9E8-16DAD802FFAC",
"TICKER.btcinr": "55E5FEB8-FEA5-416B-88FA-40211541DECA",
"TICKER.ltcjpy": "5AD8E40F-6DF3-489F-9CF1-AF28426A50CF",
"DEPTH.btccad": "5B234CC3-A7C1-47CE-854F-27AEE4CDBDA5",
"TICKER.btcnzd": "5DDD27CA-2466-4D1A-8961-615DEDB68BF1",
"DEPTH.btcgbp": "60C3AF1B-5D40-4D0E-B9FC-CCAB433D2E9C",
"DEPTH.btcnok": "66DA7FB4-6B0C-4A10-9CB7-E2944E046EB5",
"DEPTH.btcthb": "67879668-532F-41F9-8EB0-55E7593A5AB8",
"TICKER.btcsek": "6CAF1244-655B-460F-BEAF-5C56D1F4BEA7",
"TICKER.btcnok": "7532E866-3A03-4514-A4B1-6F86E3A8DC11",
"TICKER.btcgbp": "7B842B7D-D1F9-46FA-A49C-C12F1AD5A533",
"TRADE.LAG": "85174711-BE64-4DE1-B783-0628995D7914",
"DEPTH.btcsek": "8F1FEFAA-7C55-4420-ADA0-4DE15C1C38F3",
"DEPTH.btcdkk": "9219ABB0-B50C-4007-B4D2-51D1711AB19C",
"DEPTH.btcjpy": "94483E07-D797-4DD4-BC72-DC98F1FD39E3",
"TICKER.nmcusd": "9AAEFD15-D101-49F3-A2FD-6B63B85B6BED",
"TICKER.ltcaud": "A046600A-A06C-4EBF-9FFB-BDC8157227E8",
"TICKER.btcjpy": "A39AE532-6A3C-4835-AF8C-DDA54CB4874E",
"DEPTH.btcczk": "A7A970CF-4F6C-4D85-A74E-AC0979049B87",
"TICKER.ltcdkk": "B10A706E-E8C7-4EA8-9148-669F86930B36",
"TICKER.btcpln": "B4A02CB3-2E2D-4A88-AEEA-3C66CB604D01",
"TEST": "BAD99F24-FA8B-4938-BFDF-0C1831FC6665",
"TICKER.btcrub": "BD04F720-3C70-4DCE-AE71-2422AB862C65",
"TICKER.nmcgbp": "BF5126BA-5187-456F-8AE6-963678D0607F",
"TICKER.btckrw": "BF85048D-4DB9-4DBE-9CA3-5B83A1A4186E",
"TICKER.btccny": "C251EC35-56F9-40AB-A4F6-13325C349DE4",
"DEPTH.btcnzd": "CEDF8730-BCE6-4278-B6FE-9BEE42930E95",
"TICKER.btchkd": "D3AE78DD-01DD-4074-88A7-B8AA03CD28DD",
"TICKER.btcthb": "D58E3B69-9560-4B9E-8C58-B5C0F3FDA5E1",
"TICKER.btcusd": "D5F06780-30A8-4A48-A2F8-7ED181B4A13F",
"DEPTH.btcrub": "D6412CA0-B686-464C-891A-D1BA3943F3C6",
"TICKER.nmceur": "D8512D04-F262-4A14-82F2-8E5C96C15E68",
"TRADE.btc": "DBF1DEE9-4F2E-4A08-8CB7-748919A71B21",
"TICKER.nmccad": "DC28033E-7506-484C-905D-1C811A613323",
"DEPTH.btcpln": "E4FF055A-F8BF-407E-AF76-676CAD319A21",
"TICKER.btcdkk": "E5CE0604-574A-4059-9493-80AF46C776B3",
"TICKER.btcaud": "EB6AAA11-99D0-4F64-9E8C-1140872A423D"
}

See https://help.pubnub.com/entries/24113341-How-do-I-Page-Through-Stored-Messages-
Contact PubNub support (help#...) if further assistance is needed

The below Java class can be used to easily retrieve and process long time ranges of history messages in an advanced for loop of this form:
PubnubHistoryExcerpt history = new PubnubHistoryExcerpt(pubnub, channel, start, end);
for (Object message : history) {
// do something with the message object
}
The messages are retrieved on the fly, so no memory problem occurs.
Below is the complete code. You can find a fully runnable usage example in the main() method inside of the class.
I haven't yet tested this class extensively. Enhancements are welcome.
/*
* PubnubHistoryExcerpt.java
*
* This file is distributed under the FreeBSD License:
*
* Copyright (c) 2014, Daniel S. (http://stackoverflow.com/users/1838726/daniel-s)
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* The views and conclusions contained in the software and documentation are those
* of the authors and should not be interpreted as representing official policies,
* either expressed or implied, of the FreeBSD Project.
*/
import java.math.BigDecimal;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.NoSuchElementException;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.pubnub.api.Callback;
import com.pubnub.api.Pubnub;
import com.pubnub.api.PubnubError;
/**
* You can use this class to iterate over historical PubNub messages. The messages are retrieved transparently while you
* iterate over the history excerpt. This class and the returned iterators are thread-safe.
*
* See {#link #main(String[])} for a usage example.
*/
public class PubnubHistoryExcerpt implements Iterable<Object> {
/**
* This main method contains a usage example for this class. It downloads last 3 hour's messages of the MtGox BTC/USD ticker
* channel from PubNub and outputs the timestamp and USD value which are found in the messages.
*/
public static void main(String[] args) throws JSONException {
String PUBNUB_SUBSCRIBE_KEY_MTGOX = "sub-c-50d56e1e-2fd9-11e3-a041-02ee2ddab7fe";
String PUBNUB_CHANNEL_MTGOX_TICKER_BTCUSD = "d5f06780-30a8-4a48-a2f8-7ed181b4a13f";
Pubnub pubnub = new Pubnub(null, PUBNUB_SUBSCRIBE_KEY_MTGOX);
long ONE_HOUR_IN_MILLIS = 60 * 60 * 1000;
long end = System.currentTimeMillis();
long start = end - 3 * ONE_HOUR_IN_MILLIS;
// convert from milliseconds as time unit (10^-3 seconds) to
// pubnub's better-than-microsecond precision time units (10^-7 seconds)
start *= 10000;
end *= 10000;
PubnubHistoryExcerpt history = new PubnubHistoryExcerpt(pubnub, PUBNUB_CHANNEL_MTGOX_TICKER_BTCUSD, start, end);
DefaultDateFormat dateFormat = DefaultDateFormat.create();
for (Object message : history) {
JSONObject messageJson = (JSONObject) message;
JSONObject ticker = messageJson.getJSONObject("ticker");
long instant = ticker.getLong("now");
BigDecimal value = new BigDecimal(ticker.getJSONObject("last_local").getString("value"));
instant /= 1000; // convert from microseconds to milliseconds
System.out.println(dateFormat.format(instant) + ": " + value);
}
System.exit(0);
}
/**
* This is the maximum number of messages to fetch in one batch. If you fetch many messages, higher numbers improve
* performance. Setting this to a value higher than 100 doesn't have an effect, because Pubnub currently doesn't
* support fetching more than this many messages at once.
*/
private static final int BATCH_SIZE = 100;
private final Pubnub pubnub;
private final String channel;
private final long start;
private final long end;
/**
* Constructs a new excerpt over which you can iterate. Insances represent an excerpt. No retrieval operations are
* started unless you call iterator().next() for the first time.
*
* #param pubnub
* The Pubnub connection to use for retrieving messages.
* #param channel
* The channel for which to retrieve historical messages.
* #param start
* The beginning of the time interval for which to retrieve messages, in pubnub's time units (10^-7
* seconds, so milliseconds * 10000) since 1970-01-01 00:00:00).
* #param end
* The end of the time interval for which to retrieve messages, in pubnub's time units (10^-7 seconds, so
* milliseconds * 10000) since 1970-01-01 00:00:00).
*/
private PubnubHistoryExcerpt(Pubnub pubnub, String channel, long start, long end) {
this.pubnub = pubnub;
this.channel = channel;
this.start = start;
this.end = end;
}
public Iterator<Object> iterator() {
return new Iter();
}
private class Iter implements Iterator<Object> {
/**
* This list is used as a fifo buffer for messages retrieves through this iterator. It also acts as the main
* synchronization lock for synchronizing access between threads accessing this class as an iterator and threads
* calling back from the Pubnub API.
*/
private LinkedList<Object> buffer = new LinkedList<Object>();
/**
* This field stores the end of the time range of the previous batch retrieval, in Pubnub time units (10th of a
* microsecond, so milliseconds*10000). For the following batch retrieval, this is used as the start time for
* retrieving the following messages.
*/
private long prevBatchTimeRangeEnd = PubnubHistoryExcerpt.this.start;
/**
* Retrieval of messages is handled asynchronously. That means that exceptions which are thrown during retrieval
* can't automatically be propagated through to the code which invokes <code>next()</code> or
* <code>hasNext()</code> . Therefor, such an exception is stored temporarily in this field and then re-thrown
* from within <code>next()</code> or <code>hasNext()</code>.
*/
private Exception caughtDuringRetrieval = null;
/**
* This object is used to wait on and to notify about updates of the buffer.
*/
private Object notifier = new Object();
/**
* Because of spurious wakeups that can happen during wait(), this field is necessary to tell the waiting thread
* if retrieval is still running.
*/
private boolean retrieving = false;
/**
* The callback object to use for retrieving messages. This is stored in a field here for re-use. This is a
* compromise between performance and low memory footprint, slightly in favor of performance.
*/
private InternalCallback internalCallback = new InternalCallback();
private void retrieveNextBatch() {
synchronized (notifier) {
this.retrieving = true;
// String startStr = DefaultDateFormat.create().format(prevBatchTimeRangeEnd / 10000);
// String endStr = DefaultDateFormat.create().format(end / 10000);
// System.out.println("fetching from " + startStr + " till " + endStr);
if (Iter.this.prevBatchTimeRangeEnd < PubnubHistoryExcerpt.this.end) {
PubnubHistoryExcerpt.this.pubnub.history( //
PubnubHistoryExcerpt.this.channel, //
Iter.this.prevBatchTimeRangeEnd, //
PubnubHistoryExcerpt.this.end, //
BATCH_SIZE, //
false, //
Iter.this.internalCallback //
);
waitUntilNextBatchRetrievalFinished();
}
}
}
private void waitUntilNextBatchRetrievalFinished() {
while (this.retrieving) {
try {
this.notifier.wait();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
private class InternalCallback extends Callback {
#Override
public void successCallback(String channel, Object message) {
synchronized (Iter.this.notifier) {
try {
processSuccessCallback(channel, message);
} catch (Exception e) {
Iter.this.caughtDuringRetrieval = e;
} finally {
Iter.this.retrieving = false;
Iter.this.notifier.notifyAll();
}
}
}
#Override
public void errorCallback(String channel, PubnubError error) {
Iter.this.caughtDuringRetrieval = new Exception("" + //
error.getClass().getName() + ": " + //
error.getErrorString() + //
" (code=" + error.errorCode + "; extendedCode=" + error.errorCodeExtended + ")");
Iter.this.caughtDuringRetrieval.fillInStackTrace();
}
}
private void processSuccessCallback(String channel, Object message) throws JSONException {
if (message == null)
throw new NullPointerException("retrieved message is null");
if (!(message instanceof JSONArray))
throw new RuntimeException("retrieved message is not a " + JSONArray.class.getName());
JSONArray historyMessage = (JSONArray) message;
// System.out.println(historyMessage.toString(2));
JSONArray messageList = extractMessageList(historyMessage);
long batchTimeRangeEnd = extractBatchTimeRangeEnd(historyMessage);
if (batchTimeRangeEnd > 0)
Iter.this.prevBatchTimeRangeEnd = batchTimeRangeEnd;
else
Iter.this.prevBatchTimeRangeEnd = end;
processMessageList(messageList);
}
private void processMessageList(JSONArray messageList) {
int i = 0;
for (; i < messageList.length(); i++) {
JSONObject message;
try {
message = messageList.getJSONObject(i);
} catch (JSONException e) {
String str;
try {
str = messageList.toString(2);
} catch (JSONException secondaryE) {
str = "(couldn't convert messageList to String because of " + secondaryE.toString() + ")";
}
throw new RuntimeException("couldn't extract message at index " + i + " from messageList (messageList:\n" + str
+ "\n(end of messageList)\n)", e);
}
Iter.this.buffer.add(message);
}
}
private long extractBatchTimeRangeEnd(JSONArray historyMessage) {
long batchTimeRangeEnd;
try {
batchTimeRangeEnd = historyMessage.getLong(2);
} catch (JSONException e) {
String str = safeConvertHistoryMessageToString(historyMessage);
throw new RuntimeException("could not extract element 2 (batchTimeRangeEnd) of retrieved historyMessage (historyMessage:\n" + str
+ "\n(end of historyMessage)\n)", e);
}
return batchTimeRangeEnd;
}
private String safeConvertHistoryMessageToString(JSONArray historyMessage) {
String str;
try {
str = historyMessage.toString(2);
} catch (JSONException secondaryE) {
str = "(couldn't convert historyMessage to String because of " + secondaryE.toString() + ")";
}
return str;
}
private JSONArray extractMessageList(JSONArray historyMessage) {
JSONArray messageArJson;
try {
messageArJson = historyMessage.getJSONArray(0);
} catch (JSONException e) {
String str = safeConvertHistoryMessageToString(historyMessage);
throw new RuntimeException("could not extract element 0 (messageList) of retrieved historyMessage (historyMessage:\n" + str
+ "\n(end of historyMessage)\n)", e);
}
return messageArJson;
}
public boolean hasNext() {
synchronized (Iter.this.buffer) {
ensureNotInExceptionState();
if (Iter.this.buffer.isEmpty())
retrieveNextBatch();
return !Iter.this.buffer.isEmpty();
}
}
public Object next() {
synchronized (Iter.this.buffer) {
if (!hasNext()) {
throw new NoSuchElementException("there are no more elements in this iterator");
}
Object result = Iter.this.buffer.removeFirst();
return result;
}
}
private void ensureNotInExceptionState() {
if (caughtDuringRetrieval != null) {
throw new RuntimeException("an exception was caught already by a previous attempt to access this iterator", caughtDuringRetrieval);
}
}
public void remove() {
throw new UnsupportedOperationException(getClass().getName() + " doesn't support remove()");
}
}
}

Related

Chainlink API call job not fulfilled - tutorial

I have been trying to learn how to make a GET request using Chainlink.
I was watching this tutorial: https://www.youtube.com/watch?v=ay4rXZhAefs and I used the exact same code that they used in the video and I also followed the same steps using Remix (I have my metamask (Kovan) open with enough ETH and LINK and I also funded my contract with enough LINK). However after I call requestVolumeData() the volume variable doesn't update and stays 0.
I have tried to use different networks and oracles/jobs, but I seem to run into the same issue over and over again. Any help would be appreciated.
pragma solidity ^0.6.0;
import "#chainlink/contracts/src/v0.6/ChainlinkClient.sol";
/**
* THIS IS AN EXAMPLE CONTRACT WHICH USES HARDCODED VALUES FOR CLARITY.
* PLEASE DO NOT USE THIS CODE IN PRODUCTION.
*/
contract APIConsumer is ChainlinkClient {
using Chainlink for Chainlink.Request;
uint256 public volume;
address private oracle;
bytes32 private jobId;
uint256 private fee;
/**
* Network: Kovan
* Oracle: 0x2f90A6D021db21e1B2A077c5a37B3C7E75D15b7e
* Job ID: 29fa9aa13bf1468788b7cc4a500a45b8
* Fee: 0.1 LINK
*/
constructor() public {
setPublicChainlinkToken();
oracle = 0x2f90A6D021db21e1B2A077c5a37B3C7E75D15b7e;
jobId = "29fa9aa13bf1468788b7cc4a500a45b8";
fee = 0.1 * 10 ** 18; // (Varies by network and job)
}
/**
* Create a Chainlink request to retrieve API response, find the target
* data, then multiply by 1000000000000000000 (to remove decimal places from data).
*/
function requestVolumeData() public returns (bytes32 requestId)
{
Chainlink.Request memory request = buildChainlinkRequest(jobId, address(this), this.fulfill.selector);
// Set the URL to perform the GET request on
request.add("get", "https://min-api.cryptocompare.com/data/pricemultifull?fsyms=ETH&tsyms=USD");
// Set the path to find the desired data in the API response, where the response format is:
// {"RAW":
// {"ETH":
// {"USD":
// {
// "VOLUME24HOUR": xxx.xxx,
// }
// }
// }
// }
request.add("path", "RAW.ETH.USD.VOLUME24HOUR");
// Multiply the result by 1000000000000000000 to remove decimals
int timesAmount = 10**18;
request.addInt("times", timesAmount);
// Sends the request
return sendChainlinkRequestTo(oracle, request, fee);
}
/**
* Receive the response in the form of uint256
*/
function fulfill(bytes32 _requestId, uint256 _volume) public recordChainlinkFulfillment(_requestId)
{
volume = _volume;
}
// function withdrawLink() external {} - Implement a withdraw function to avoid locking your LINK in the contract
}
Are you giving some time to let the node fetch the request? When you do this example, you usually have to click the requestVolumeData() button, multiple times, waiting for the node to fulfill the request. Less than two minutes usually.
Here's another more recent version of that example contract. It's nearly the same.
// SPDX-License-Identifier: MIT
pragma solidity ^0.6.6;
import "#chainlink/contracts/src/v0.6/ChainlinkClient.sol";
contract APIConsumer is ChainlinkClient {
uint256 public volume;
address private oracle;
bytes32 private jobId;
uint256 private fee;
/**
* Network: Kovan
* Oracle: 0x2f90A6D021db21e1B2A077c5a37B3C7E75D15b7e
* Job ID: 29fa9aa13bf1468788b7cc4a500a45b8
* Fee: 0.1 LINK
*/
constructor(address _oracle, string memory _jobId, uint256 _fee, address _link) public {
if (_link == address(0)) {
setPublicChainlinkToken();
} else {
setChainlinkToken(_link);
}
// oracle = 0x2f90A6D021db21e1B2A077c5a37B3C7E75D15b7e;
// jobId = "29fa9aa13bf1468788b7cc4a500a45b8";
// fee = 0.1 * 10 ** 18; // 0.1 LINK
oracle = _oracle;
jobId = stringToBytes32(_jobId);
fee = _fee;
}
/**
* Create a Chainlink request to retrieve API response, find the target
* data, then multiply by 1000000000000000000 (to remove decimal places from data).
*/
function requestVolumeData() public returns (bytes32 requestId)
{
Chainlink.Request memory request = buildChainlinkRequest(jobId, address(this), this.fulfill.selector);
// Set the URL to perform the GET request on
request.add("get", "https://min-api.cryptocompare.com/data/pricemultifull?fsyms=ETH&tsyms=USD");
// Set the path to find the desired data in the API response, where the response format is:
// {"RAW":
// {"ETH":
// {"USD":
// {
// "VOLUME24HOUR": xxx.xxx,
// }
// }
// }
// }
request.add("path", "RAW.ETH.USD.VOLUME24HOUR");
// Multiply the result by 1000000000000000000 to remove decimals
int timesAmount = 10**18;
request.addInt("times", timesAmount);
// Sends the request
return sendChainlinkRequestTo(oracle, request, fee);
}
/**
* Receive the response in the form of uint256
*/
function fulfill(bytes32 _requestId, uint256 _volume) public recordChainlinkFulfillment(_requestId)
{
volume = _volume;
}
function stringToBytes32(string memory source) public pure returns (bytes32 result) {
bytes memory tempEmptyStringTest = bytes(source);
if (tempEmptyStringTest.length == 0) {
return 0x0;
}
assembly {
result := mload(add(source, 32))
}
}
}

How to filter on Apache Edgent and also show the values which were filtered?

I am using Apache Edgent (Java framework) to poll values from a HCSR04 ultrasonic sensor on a Raspberry Pi every 3 seconds. I use a filter to not get values from 50cm to 80cm.
UltrasonicStream sensor = new UltrasonicStream();
DirectProvider dp = new DirectProvider();
Topology topology = dp.newTopology();
TStream<Double> tempReadings = topology.poll(sensor, 3, TimeUnit.SECONDS);
TStream<Double> filteredReadings = tempReadings.filter(reading -> reading < 50 || reading > 80);
System.out.println("filter added: tempReadings.filter(reading -> reading < 50 || reading > 80);");
filteredReadings.print();
dp.submit(topology);
I want to show some message when the values are filtered. When the values do not match with my filter I can poll them, but when they match I am not returning, that is ok. However, I want just to show that a value was filtered using Apache Edgent libraries. I know that I can do something on the public double get() method, but I wonder if I could do this trick with some method of the Apache Edgent.
public class UltrasonicStream implements Supplier {
private static final long serialVersionUID = -6511218542753341056L;
private static GpioPinDigitalOutput sensorTriggerPin;
private static GpioPinDigitalInput sensorEchoPin;
private static final GpioController gpio = GpioFactory.getInstance();
private double currentDistance = -1.0;
/**
* The HCSR04 Ultrasonic sensor is connected on the physical pin 16 and 18 which
* correspond to the GPIO 04 and 05 of the WiringPi library.
*/
public UltrasonicStream() {
// Trigger pin as OUTPUT
sensorTriggerPin = gpio.provisionDigitalOutputPin(RaspiPin.GPIO_04);
// Echo pin as INPUT
sensorEchoPin = gpio.provisionDigitalInputPin(RaspiPin.GPIO_05, PinPullResistance.PULL_DOWN);
}
/**
* This is the override method of the Supplier interface from Apache Edgent
*/
#Override
public Double get() {
try {
System.out.print("Distance in centimeters: ");
currentDistance = getDistance();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return currentDistance;
}
/**
* Retrieve the distance measured by the HCSR04 Ultrasonic sensor connected on a
* Raspberry Pi 3+B
*
* #return the distance in centimeters
* #throws InterruptedException
*/
public double getDistance() throws InterruptedException {
double distanceCM = -1;
try {
// Thread.sleep(2000);
sensorTriggerPin.high(); // Make trigger pin HIGH
Thread.sleep((long) 0.01);// Delay for 10 microseconds
sensorTriggerPin.low(); // Make trigger pin LOW
// Wait until the ECHO pin gets HIGH
while (sensorEchoPin.isLow()) {
}
// Store the current time to calculate ECHO pin HIGH time.
long startTime = System.nanoTime();
// Wait until the ECHO pin gets LOW
while (sensorEchoPin.isHigh()) {
}
// Store the echo pin HIGH end time to calculate ECHO pin HIGH time.
long endTime = System.nanoTime();
distanceCM = ((((endTime - startTime) / 1e3) / 2) / 29.1);
// Printing out the distance in centimeters
// System.out.println("Distance: " + distanceCM + " centimeters");
return distanceCM;
} catch (InterruptedException e) {
e.printStackTrace();
}
return distanceCM;
}
}
You can use TStream.split() to create two streams: one whose tuples match your filter predicate and one for those that don't. You can then do whatever you want with either stream. e.g. TStream.peek(t -> System.out.println("excluded: " + t)), or TStream.print(...)
I implemented like this:
UltrasonicStream sensor = new UltrasonicStream();
DirectProvider dp = new DirectProvider();
Topology topology = dp.newTopology();
TStream<Double> tempReadings = topology.poll(sensor, 3, TimeUnit.SECONDS);
TStream<Double> filteredReadings = tempReadings.filter(reading -> {
boolean threshold = reading < 20 || reading > 80;
if (!threshold) {
System.out.println(String.format("Threshold reached: %s cm", reading));
}
return threshold;
});
filteredReadings.print();
dp.submit(topology);

Programmatically create a new Person having an Internal Login

I am trying to programmatically create Users with Internal Accounts as part of a testing system. The following code can not create an InternalLogin because there is not password hash set at object creation time.
Can a person + internal account be created using metadata_* functions ?
data _null_;
length uri_Person uri_PW uri_IL $256;
call missing (of uri:);
rc = metadata_getnobj ("Person?#Name='testbot02'", 1, uri_Person); msg=sysmsg();
put 'NOTE: Get Person, ' rc= uri_Person= / msg;
if rc = -4 then do;
rc = metadata_newobj ('Person', uri_Person, 'testbot02'); msg=sysmsg();
put 'NOTE: New Person, ' rc= uri_Person= / msg;
end;
rc = metadata_setattr (uri_Person, 'DisplayName', 'Test Bot #2'); msg=sysmsg();
put 'NOTE: SetAttr, ' rc= / msg;
rc = metadata_newobj ('InternalLogin', uri_IL, 'autobot - IL', 'Foundation', uri_Person, 'InternalLoginInfo'); msg=sysmsg();
put 'NOTE: New InternalLogin, ' rc= / msg;
run;
Logs
NOTE: Get Person, rc=-4 uri_Person=
NOTE: New Person, rc=0 uri_Person=OMSOBJ:Person\A5OJU4RB.AP0000SX
NOTE: SetAttr, rc=0
NOTE: New InternalLogin, rc=-2
ERROR: AddMetadata of InternalLogin is missing required property PasswordHash.
NOTE: DATA statement used (Total process time):
real time 0.02 seconds
cpu time 0.01 seconds
The management console and metabrowse were used to find what objects might need to be created.
Metabrowse
I ended up using Java to create new users.
A ISecurity_1_1 instance from MakeISecurityConnection() on the metadata connection was used to SetInternalPassword
Java ended up being a lot more clear coding wise.
package sandbox.sas.metadata;
import com.sas.iom.SASIOMDefs.GenericError;
import com.sas.metadata.remote.*;
import com.sas.meta.SASOMI.*;
import java.rmi.RemoteException;
import java.util.List;
/**
*
* #author Richard
*/
public class Main {
public static final String TESTGROUPNAME = "Automated Test Users";
/**
* #param args the command line arguments
*/
public static void main(String[] args) {
String metaserver="################";
String metaport="8561";
String omrUser="sasadm#saspw";
String omrPass="##########";
try
{
MdFactoryImpl metadata = new MdFactoryImpl(false);
metadata.makeOMRConnection(metaserver,metaport,omrUser,omrPass);
MdOMIUtil util = metadata.getOMIUtil();
ISecurity_1_1 security = metadata.getConnection().MakeISecurityConnection();
MdObjectStore workunit = metadata.createObjectStore();
String foundationID = util.getFoundationReposID();
String foundationShortID = foundationID.substring(foundationID.indexOf(".") + 1);
// Create group for test bot accounts
List identityGroups = util.getMetadataObjects(foundationID, MetadataObjects.IDENTITYGROUP, MdOMIUtil.OMI_XMLSELECT,
String.format("<XMLSelect search=\"IdentityGroup[#Name='%s']\"/>", TESTGROUPNAME));
IdentityGroup identityGroup;
if (identityGroups.isEmpty()) {
identityGroup = (IdentityGroup) metadata.createComplexMetadataObject (workunit, TESTGROUPNAME, MetadataObjects.IDENTITYGROUP, foundationShortID);
identityGroup.setDisplayName(TESTGROUPNAME);
identityGroup.setDesc("Group for Automated Test Users performing concurrent Stored Process execution");
identityGroup.updateMetadataAll();
}
identityGroups = util.getMetadataObjectsSubset(workunit, foundationID, MetadataObjects.IDENTITYGROUP, MdOMIUtil.OMI_XMLSELECT,
String.format("<XMLSelect search=\"IdentityGroup[#Name='%s']\"/>", TESTGROUPNAME));
identityGroup = (IdentityGroup) identityGroups.get(0);
// Create (or Update) test bot accounts
for (int index = 1; index <= 25; index++)
{
String token = String.format("%02d", index);
String personName = String.format("testbot%s", token);
String password = String.format("testbot%s%s", token, token); * simple dynamically generated password for user (for testing scripts only);
String criteria = String.format("Person[#Name='%s']", personName);
String xmlSelect = String.format("<XMLSelect search=\"%s\"/>", criteria);
List persons = util.getMetadataObjectsSubset(workunit, foundationID, MetadataObjects.PERSON, MdOMIUtil.OMI_XMLSELECT | MdOMIUtil.OMI_GET_METADATA | MdOMIUtil.OMI_ALL_SIMPLE, xmlSelect);
Person person;
if (persons.size() == 1)
{
person = (Person) persons.get(0);
System.out.println(String.format("Have %s %s", person.getName(), person.getDisplayName()));
}
else
{
person = (Person) metadata.createComplexMetadataObject (workunit, personName, MetadataObjects.PERSON, foundationShortID);
person.setDisplayName(String.format("Test Bot #%s", token));
person.setDesc("Internal account for testing purposes");
System.out.println(String.format("Make %s, %s (%s)", person.getName(), person.getDisplayName(), person.getDesc()));
person.updateMetadataAll();
}
security.SetInternalPassword(personName, password);
AssociationList personGroups = person.getIdentityGroups();
personGroups.add(identityGroup);
person.setIdentityGroups(personGroups);
person.updateMetadataAll();
}
workunit.dispose();
metadata.closeOMRConnection();
metadata.dispose();
}
catch (GenericError | MdException | RemoteException e)
{
System.out.println(e.getLocalizedMessage());
System.exit(1);
}
}
}

How do I debug a form which does not send an email?

I have a form build from the kohana framework which should send an email.
when I press the "send" button, everything seems to work fine...no error messages...but no email turns up!
If I can´t see anything in firebug..where can I look?
the application logs state
"error: Missing i18n entry contact.captcha.valid for language en_US"
but I don´t know how to get to the bottom of the problem..any help welcome..
yours,
Rob
Ill try and find out which version im using.....the application is the latest version of Ushahidi (2.2.1) www.ushahidi.com
<?php defined('SYSPATH') OR die('No direct access allowed.');
/**
* Captcha library.
*
* $Id: Captcha.php 3917 2009-01-21 03:06:22Z zombor $
*
* #package Captcha
* #author Kohana Team
* #copyright (c) 2007-2008 Kohana Team
* #license http://kohanaphp.com/license.html
*/
class Captcha_Core {
// Captcha singleton
protected static $instance;
// Style-dependent Captcha driver
protected $driver;
// Config values
public static $config = array
(
'style' => 'basic',
'width' => 150,
'height' => 50,
'complexity' => 4,
'background' => '',
'fontpath' => '',
'fonts' => array(),
'promote' => FALSE,
);
/**
* Singleton instance of Captcha.
*
* #return object
*/
public static function instance()
{
// Create the instance if it does not exist
empty(self::$instance) and new Captcha;
return self::$instance;
}
/**
* Constructs and returns a new Captcha object.
*
* #param string config group name
* #return object
*/
public static function factory($group = NULL)
{
return new Captcha($group);
}
/**
* Constructs a new Captcha object.
*
* #throws Kohana_Exception
* #param string config group name
* #return void
*/
public function __construct($group = NULL)
{
// Create a singleton instance once
empty(self::$instance) and self::$instance = $this;
// No config group name given
if ( ! is_string($group))
{
$group = 'default';
}
// Load and validate config group
if ( ! is_array($config = Kohana::config('captcha.'.$group)))
throw new Kohana_Exception('captcha.undefined_group', $group);
// All captcha config groups inherit default config group
if ($group !== 'default')
{
// Load and validate default config group
if ( ! is_array($default = Kohana::config('captcha.default')))
throw new Kohana_Exception('captcha.undefined_group', 'default');
// Merge config group with default config group
$config += $default;
}
// Assign config values to the object
foreach ($config as $key => $value)
{
if (array_key_exists($key, self::$config))
{
self::$config[$key] = $value;
}
}
// Store the config group name as well, so the drivers can access it
self::$config['group'] = $group;
// If using a background image, check if it exists
if ( ! empty($config['background']))
{
self::$config['background'] = str_replace('\\', '/', realpath($config['background']));
if ( ! is_file(self::$config['background']))
throw new Kohana_Exception('captcha.file_not_found', self::$config['background']);
}
// If using any fonts, check if they exist
if ( ! empty($config['fonts']))
{
self::$config['fontpath'] = str_replace('\\', '/', realpath($config['fontpath'])).'/';
foreach ($config['fonts'] as $font)
{
if ( ! is_file(self::$config['fontpath'].$font))
throw new Kohana_Exception('captcha.file_not_found', self::$config['fontpath'].$font);
}
}
// Set driver name
$driver = 'Captcha_'.ucfirst($config['style']).'_Driver';
// Load the driver
if ( ! Kohana::auto_load($driver))
throw new Kohana_Exception('core.driver_not_found', $config['style'], get_class($this));
// Initialize the driver
$this->driver = new $driver;
// Validate the driver
if ( ! ($this->driver instanceof Captcha_Driver))
throw new Kohana_Exception('core.driver_implements', $config['style'], get_class($this), 'Captcha_Driver');
Kohana::log('debug', 'Captcha Library initialized');
}
/**
* Validates a Captcha response and updates response counter.
*
* #param string captcha response
* #return boolean
*/
public static function valid($response)
{
// Maximum one count per page load
static $counted;
// User has been promoted, always TRUE and don't count anymore
if (self::instance()->promoted())
return TRUE;
// Challenge result
$result = (bool) self::instance()->driver->valid($response);
// Increment response counter
if ($counted !== TRUE)
{
$counted = TRUE;
// Valid response
if ($result === TRUE)
{
self::instance()->valid_count(Session::instance()->get('captcha_valid_count') + 1);
}
// Invalid response
else
{
self::instance()->invalid_count(Session::instance()->get('captcha_invalid_count') + 1);
}
}
return $result;
}
/**
* Gets or sets the number of valid Captcha responses for this session.
*
* #param integer new counter value
* #param boolean trigger invalid counter (for internal use only)
* #return integer counter value
*/
public function valid_count($new_count = NULL, $invalid = FALSE)
{
// Pick the right session to use
$session = ($invalid === TRUE) ? 'captcha_invalid_count' : 'captcha_valid_count';
// Update counter
if ($new_count !== NULL)
{
$new_count = (int) $new_count;
// Reset counter = delete session
if ($new_count < 1)
{
Session::instance()->delete($session);
}
// Set counter to new value
else
{
Session::instance()->set($session, (int) $new_count);
}
// Return new count
return (int) $new_count;
}
// Return current count
return (int) Session::instance()->get($session);
}
/**
* Gets or sets the number of invalid Captcha responses for this session.
*
* #param integer new counter value
* #return integer counter value
*/
public function invalid_count($new_count = NULL)
{
return $this->valid_count($new_count, TRUE);
}
/**
* Resets the Captcha response counters and removes the count sessions.
*
* #return void
*/
public function reset_count()
{
$this->valid_count(0);
$this->valid_count(0, TRUE);
}
/**
* Checks whether user has been promoted after having given enough valid responses.
*
* #param integer valid response count threshold
* #return boolean
*/
public function promoted($threshold = NULL)
{
// Promotion has been disabled
if (self::$config['promote'] === FALSE)
return FALSE;
// Use the config threshold
if ($threshold === NULL)
{
$threshold = self::$config['promote'];
}
// Compare the valid response count to the threshold
return ($this->valid_count() >= $threshold);
}
/**
* Returns or outputs the Captcha challenge.
*
* #param boolean TRUE to output html, e.g. <img src="#" />
* #return mixed html string or void
*/
public function render($html = TRUE)
{
return $this->driver->render($html);
}
/**
* Magically outputs the Captcha challenge.
*
* #return mixed
*/
public function __toString()
{
return $this->render();
}
} // End Captcha Class
From the little details provided I think your site uses the I18n library from Kohana for internationalization of the site. The other option is your site use Kohana messages for showing form errors.
The Validation class uses the __() function internally when generation validation errors. I think you don't have a message for key valid specified in APPPATH/messages/contact/captcha.php.
You should try to investigate more on how the form is processed and whether some validation errors are being generated. There might be an error in the captcha validation and despite it seem there is no error, there might be and just the error message could not be shown.
class Contact_Controller extends Main_Controller
{
function __construct()
{
parent::__construct();
}
public function index()
{
$this->template->header->this_page = 'contact';
$this->template->content = new View('contact');
$this->template->header->page_title .= Kohana::lang('ui_main.contact').Kohana::config('settings.title_delimiter');
// Setup and initialize form field names
$form = array (
'contact_name' => '',
'contact_email' => '',
'contact_phone' => '',
'contact_subject' => '',
'contact_message' => '',
'captcha' => ''
);
// Copy the form as errors, so the errors will be stored with keys
// corresponding to the form field names
$captcha = Captcha::factory();
$errors = $form;
$form_error = FALSE;
$form_sent = FALSE;
// Check, has the form been submitted, if so, setup validation
if ($_POST)
{
// Instantiate Validation, use $post, so we don't overwrite $_POST fields with our own things
$post = Validation::factory($_POST);
// Add some filters
$post->pre_filter('trim', TRUE);
// Add some rules, the input field, followed by a list of checks, carried out in order
$post->add_rules('contact_name', 'required', 'length[3,100]');
$post->add_rules('contact_email', 'required','email', 'length[4,100]');
$post->add_rules('contact_subject', 'required', 'length[3,100]');
$post->add_rules('contact_message', 'required');
$post->add_rules('captcha', 'required', 'Captcha::valid');
// Test to see if things passed the rule checks
if ($post->validate())
{
// Yes! everything is valid - Send email
$site_email = Kohana::config('settings.site_email');
$message = Kohana::lang('ui_admin.sender').": " . $post->contact_name . "\n";
$message .= Kohana::lang('ui_admin.email').": " . $post->contact_email . "\n";
$message .= Kohana::lang('ui_admin.phone').": " . $post->contact_phone . "\n\n";
$message .= Kohana::lang('ui_admin.message').": \n" . $post->contact_message . "\n\n\n";
$message .= "~~~~~~~~~~~~~~~~~~~~~~\n";
$message .= Kohana::lang('ui_admin.sent_from_website'). url::base();
// Send Admin Message
email::send( $site_email, $post->contact_email, $post->contact_subject, $message, FALSE );
$form_sent = TRUE;
}
// No! We have validation errors, we need to show the form again, with the errors
else
{
// repopulate the form fields
$form = arr::overwrite($form, $post->as_array());
// populate the error fields, if any
$errors = arr::overwrite($errors, $post->errors('contact'));
$form_error = TRUE;
}
}
$this->template->content->form = $form;
$this->template->content->errors = $errors;
$this->template->content->form_error = $form_error;
$this->template->content->form_sent = $form_sent;
$this->template->content->captcha = $captcha;
// Rebuild Header Block
$this->template->header->header_block = $this->themes->header_block();
$this->template->footer->footer_block = $this->themes->footer_block();
}
}
My problem has been solved!
My Email server was not set up properly. I resolved the postfix issue and now the comments form is working as it should,
thanks to all who answered,
Rob

Player with custom data source on Blackberry

I must create a custom media player within the application with support for mp3 and wav files. I read in the documentation I can't seek or get the media file duration without a custom datasource.
I checked the demo in the JDE 4.6 but I have still problems... I can't get the duration, it returns much more then expected so I'm sure I screwed up something while I modified the code to read the mp3 file locally from the filesystem.
Can somebody tell me what I did wrong? (I can hear the mp3, so the player plays it correctly from start to end)
I must support OSs >= 4.6.
Here is my modified datasource:
/* LimitedRateStreaminSource.java
*
* Copyright © 1998-2009 Research In Motion Ltd.
*
* Note: For the sake of simplicity, this sample application may not leverage
* resource bundles and resource strings. However, it is STRONGLY recommended
* that application developers make use of the localization features available
* within the BlackBerry development platform to ensure a seamless application
* experience across a variety of languages and geographies.
* For more information on localizing your application, please refer to the
* BlackBerry Java Development Environment Development Guide associated with
* this release.
*/
package com.halcyon.tawkwidget.model;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import javax.microedition.io.Connector;
import javax.microedition.io.file.FileConnection;
import javax.microedition.media.Control;
import javax.microedition.media.protocol.ContentDescriptor;
import javax.microedition.media.protocol.DataSource;
import javax.microedition.media.protocol.SourceStream;
import net.rim.device.api.io.SharedInputStream;
/**
* The data source used by the BufferedPlayback's media player.
*/
public final class LimitedRateStreamingSource extends DataSource
{
/** The max size to be read from the stream at one time. */
private static final int READ_CHUNK = 512; // bytes
/** A reference to the field which displays the load status. */
//private TextField _loadStatusField;
/** A reference to the field which displays the player status. */
//private TextField _playStatusField;
/**
* The minimum number of bytes that must be buffered before the media file
* will begin playing.
*/
private int _startBuffer = 200000;
/** The maximum size (in bytes) of a single read. */
private int _readLimit = 32000;
/**
* The minimum forward byte buffer which must be maintained in order for
* the video to keep playing. If the forward buffer falls below this
* number, the playback will pause until the buffer increases.
*/
private int _pauseBytes = 64000;
/**
* The minimum forward byte buffer required to resume
* playback after a pause.
*/
private int _resumeBytes = 128000;
/** The stream connection over which media content is passed. */
//private ContentConnection _contentConnection;
private FileConnection _fileConnection;
/** An input stream shared between several readers. */
private SharedInputStream _readAhead;
/** A stream to the buffered resource. */
private LimitedRateSourceStream _feedToPlayer;
/** The MIME type of the remote media file. */
private String _forcedContentType;
/** A counter for the total number of buffered bytes */
private volatile int _totalRead;
/** A flag used to tell the connection thread to stop */
private volatile boolean _stop;
/**
* A flag used to indicate that the initial buffering is complete. In
* other words, that the current buffer is larger than the defined start
* buffer size.
*/
private volatile boolean _bufferingComplete;
/** A flag used to indicate that the remote file download is complete. */
private volatile boolean _downloadComplete;
/** The thread which retrieves the remote media file. */
private ConnectionThread _loaderThread;
/** The local save file into which the remote file is written. */
private FileConnection _saveFile;
/** A stream for the local save file. */
private OutputStream _saveStream;
/**
* Constructor.
* #param locator The locator that describes the DataSource.
*/
public LimitedRateStreamingSource(String locator)
{
super(locator);
}
/**
* Open a connection to the locator.
* #throws IOException
*/
public void connect() throws IOException
{
//Open the connection to the remote file.
_fileConnection = (FileConnection)Connector.open(getLocator(),
Connector.READ);
//Cache a reference to the locator.
String locator = getLocator();
//Report status.
System.out.println("Loading: " + locator);
//System.out.println("Size: " + _contentConnection.getLength());
System.out.println("Size: " + _fileConnection.totalSize());
//The name of the remote file begins after the last forward slash.
int filenameStart = locator.lastIndexOf('/');
//The file name ends at the first instance of a semicolon.
int paramStart = locator.indexOf(';');
//If there is no semicolon, the file name ends at the end of the line.
if (paramStart < 0)
{
paramStart = locator.length();
}
//Extract the file name.
String filename = locator.substring(filenameStart, paramStart);
System.out.println("Filename: " + filename);
//Open a local save file with the same name as the remote file.
_saveFile = (FileConnection) Connector.open("file:///SDCard"+
"/blackberry/music" + filename, Connector.READ_WRITE);
//If the file doesn't already exist, create it.
if (!_saveFile.exists())
{
_saveFile.create();
}
System.out.println("---------- 1");
//Open the file for writing.
_saveFile.setReadable(true);
//Open a shared input stream to the local save file to
//allow many simultaneous readers.
SharedInputStream fileStream = SharedInputStream.getSharedInputStream(
_saveFile.openInputStream());
//Begin reading at the beginning of the file.
fileStream.setCurrentPosition(0);
System.out.println("---------- 2");
//If the local file is smaller than the remote file...
if (_saveFile.fileSize() < _fileConnection.totalSize())
{
System.out.println("---------- 3");
//Did not get the entire file, set the system to try again.
_saveFile.setWritable(true);
System.out.println("---------- 4");
//A non-null save stream is used as a flag later to indicate that
//the file download was incomplete.
_saveStream = _saveFile.openOutputStream();
System.out.println("---------- 5");
//Use a new shared input stream for buffered reading.
_readAhead = SharedInputStream.getSharedInputStream(
_fileConnection.openInputStream());
System.out.println("---------- 6");
}
else
{
//The download is complete.
System.out.println("---------- 7");
_downloadComplete = true;
//We can use the initial input stream to read the buffered media.
_readAhead = fileStream;
System.out.println("---------- 8");
//We can close the remote connection.
_fileConnection.close();
System.out.println("---------- 9");
}
if (_forcedContentType != null)
{
//Use the user-defined content type if it is set.
System.out.println("---------- 10");
_feedToPlayer = new LimitedRateSourceStream(_readAhead,
_forcedContentType);
System.out.println("---------- 11");
}
else
{
System.out.println("---------- 12");
//Otherwise, use the MIME types of the remote file.
// _feedToPlayer = new LimitedRateSourceStream(_readAhead,
_fileConnection));
}
System.out.println("---------- 13");
}
/**
* Destroy and close all existing connections.
*/
public void disconnect() {
try
{
if (_saveStream != null)
{
//Destroy the stream to the local save file.
_saveStream.close();
_saveStream = null;
}
//Close the local save file.
_saveFile.close();
if (_readAhead != null)
{
//Close the reader stream.
_readAhead.close();
_readAhead = null;
}
//Close the remote file connection.
_fileConnection.close();
//Close the stream to the player.
_feedToPlayer.close();
}
catch (Exception e)
{
System.err.println(e.getMessage());
}
}
/**
* Returns the content type of the remote file.
* #return The content type of the remote file.
*/
public String getContentType()
{
return _feedToPlayer.getContentDescriptor().getContentType();
}
/**
* Returns a stream to the buffered resource.
* #return A stream to the buffered resource.
*/
public SourceStream[] getStreams()
{
return new SourceStream[] { _feedToPlayer };
}
/**
* Starts the connection thread used to download the remote file.
*/
public void start() throws IOException
{
//If the save stream is null, we have already completely downloaded
//the file.
if (_saveStream != null)
{
//Open the connection thread to finish downloading the file.
_loaderThread = new ConnectionThread();
_loaderThread.start();
}
}
/**
* Stop the connection thread.
*/
public void stop() throws IOException
{
//Set the boolean flag to stop the thread.
_stop = true;
}
/**
* #see javax.microedition.media.Controllable#getControl(String)
*/
public Control getControl(String controlType)
{
// No implemented Controls.
return null;
}
/**
* #see javax.microedition.media.Controllable#getControls()
*/
public Control[] getControls()
{
// No implemented Controls.
return null;
}
/**
* Force the lower level stream to a given content type. Must be called
* before the connect function in order to work.
* #param contentType The content type to use.
*/
public void setContentType(String contentType)
{
_forcedContentType = contentType;
}
/**
* A stream to the buffered media resource.
*/
private final class LimitedRateSourceStream implements SourceStream
{
/** A stream to the local copy of the remote resource. */
private SharedInputStream _baseSharedStream;
/** Describes the content type of the media file. */
private ContentDescriptor _contentDescriptor;
/**
* Constructor. Creates a LimitedRateSourceStream from
* the given InputStream.
* #param inputStream The input stream used to create a new reader.
* #param contentType The content type of the remote file.
*/
LimitedRateSourceStream(InputStream inputStream, String contentType)
{
System.out.println("[LimitedRateSoruceStream]---------- 1");
_baseSharedStream = SharedInputStream.getSharedInputStream(
inputStream);
System.out.println("[LimitedRateSoruceStream]---------- 2");
_contentDescriptor = new ContentDescriptor(contentType);
System.out.println("[LimitedRateSoruceStream]---------- 3");
}
/**
* Returns the content descriptor for this stream.
* #return The content descriptor for this stream.
*/
public ContentDescriptor getContentDescriptor()
{
return _contentDescriptor;
}
/**
* Returns the length provided by the connection.
* #return long The length provided by the connection.
*/
public long getContentLength()
{
return _fileConnection.totalSize();
}
/**
* Returns the seek type of the stream.
*/
public int getSeekType()
{
return RANDOM_ACCESSIBLE;
//return SEEKABLE_TO_START;
}
/**
* Returns the maximum size (in bytes) of a single read.
*/
public int getTransferSize()
{
return _readLimit;
}
/**
* Writes bytes from the buffer into a byte array for playback.
* #param bytes The buffer into which the data is read.
* #param off The start offset in array b at which the data is written.
* #param len The maximum number of bytes to read.
* #return the total number of bytes read into the buffer, or -1 if
* there is no more data because the end of the stream has been reached.
* #throws IOException
*/
public int read(byte[] bytes, int off, int len) throws IOException
{
System.out.println("[LimitedRateSoruceStream]---------- 5");
System.out.println("Read Request for: " + len + " bytes");
//Limit bytes read to our readLimit.
int readLength = len;
System.out.println("[LimitedRateSoruceStream]---------- 6");
if (readLength > getReadLimit())
{
readLength = getReadLimit();
}
//The number of available byes in the buffer.
int available;
//A boolean flag indicating that the thread should pause
//until the buffer has increased sufficiently.
boolean paused = false;
System.out.println("[LimitedRateSoruceStream]---------- 7");
for (;;)
{
available = _baseSharedStream.available();
System.out.println("[LimitedRateSoruceStream]---------- 8");
if (_downloadComplete)
{
//Ignore all restrictions if downloading is complete.
System.out.println("Complete, Reading: " + len +
" - Available: " + available);
return _baseSharedStream.read(bytes, off, len);
}
else if(_bufferingComplete)
{
if (paused && available > getResumeBytes())
{
//If the video is paused due to buffering, but the
//number of available byes is sufficiently high,
//resume playback of the media.
System.out.println("Resuming - Available: " +
available);
paused = false;
return _baseSharedStream.read(bytes, off, readLength);
}
else if(!paused && (available > getPauseBytes() ||
available > readLength))
{
//We have enough information for this media playback.
if (available < getPauseBytes())
{
//If the buffer is now insufficient, set the
//pause flag.
paused = true;
}
System.out.println("Reading: " + readLength +
" - Available: " + available);
return _baseSharedStream.read(bytes, off, readLength);
}
else if(!paused)
{
//Set pause until loaded enough to resume.
paused = true;
}
}
else
{
//We are not ready to start yet, try sleeping to allow the
//buffer to increase.
try
{
Thread.sleep(500);
}
catch (Exception e)
{
System.err.println(e.getMessage());
}
}
}
}
/**
* #see javax.microedition.media.protocol.SourceStream#seek(long)
*/
public long seek(long where) throws IOException
{
_baseSharedStream.setCurrentPosition((int) where);
return _baseSharedStream.getCurrentPosition();
}
/**
* #see javax.microedition.media.protocol.SourceStream#tell()
*/
public long tell()
{
return _baseSharedStream.getCurrentPosition();
}
/**
* Close the stream.
* #throws IOException
*/
void close() throws IOException
{
_baseSharedStream.close();
}
/**
* #see javax.microedition.media.Controllable#getControl(String)
*/
public Control getControl(String controlType)
{
// No implemented controls.
return null;
}
/**
* #see javax.microedition.media.Controllable#getControls()
*/
public Control[] getControls()
{
// No implemented controls.
return null;
}
}
/**
* A thread which downloads the remote file and writes it to the local file.
*/
private final class ConnectionThread extends Thread
{
/**
* Download the remote media file, then write it to the local
* file.
* #see java.lang.Thread#run()
*/
public void run()
{
try
{
byte[] data = new byte[READ_CHUNK];
int len = 0;
//Until we reach the end of the file.
while (-1 != (len = _readAhead.read(data)))
{
_totalRead += len;
if (!_bufferingComplete && _totalRead > getStartBuffer())
{
//We have enough of a buffer to begin playback.
_bufferingComplete = true;
System.out.println("Initial Buffering Complete");
}
if (_stop)
{
//Stop reading.
return;
}
}
System.out.println("Downloading Complete");
System.out.println("Total Read: " + _totalRead);
//If the downloaded data is not the same size
//as the remote file, something is wrong.
if (_totalRead != _fileConnection.totalSize())
{
System.err.println("* Unable to Download entire file *");
}
_downloadComplete = true;
_readAhead.setCurrentPosition(0);
//Write downloaded data to the local file.
while (-1 != (len = _readAhead.read(data)))
{
_saveStream.write(data);
}
}
catch (Exception e)
{
System.err.println(e.toString());
}
}
}
/**
* Gets the minimum forward byte buffer which must be maintained in
* order for the video to keep playing.
* #return The pause byte buffer.
*/
int getPauseBytes()
{
return _pauseBytes;
}
/**
* Sets the minimum forward buffer which must be maintained in order
* for the video to keep playing.
* #param pauseBytes The new pause byte buffer.
*/
void setPauseBytes(int pauseBytes)
{
_pauseBytes = pauseBytes;
}
/**
* Gets the maximum size (in bytes) of a single read.
* #return The maximum size (in bytes) of a single read.
*/
int getReadLimit()
{
return _readLimit;
}
/**
* Sets the maximum size (in bytes) of a single read.
* #param readLimit The new maximum size (in bytes) of a single read.
*/
void setReadLimit(int readLimit)
{
_readLimit = readLimit;
}
/**
* Gets the minimum forward byte buffer required to resume
* playback after a pause.
* #return The resume byte buffer.
*/
int getResumeBytes()
{
return _resumeBytes;
}
/**
* Sets the minimum forward byte buffer required to resume
* playback after a pause.
* #param resumeBytes The new resume byte buffer.
*/
void setResumeBytes(int resumeBytes)
{
_resumeBytes = resumeBytes;
}
/**
* Gets the minimum number of bytes that must be buffered before the
* media file will begin playing.
* #return The start byte buffer.
*/
int getStartBuffer()
{
return _startBuffer;
}
/**
* Sets the minimum number of bytes that must be buffered before the
* media file will begin playing.
* #param startBuffer The new start byte buffer.
*/
void setStartBuffer(int startBuffer)
{
_startBuffer = startBuffer;
}
}
And in this way i use it:
LimitedRateStreamingSource source = new
LimitedRateStreamingSource("file:///SDCard/music3.mp3");
source.setContentType("audio/mpeg");
mediaPlayer = javax.microedition.media.Manager.createPlayer(source);
mediaPlayer.addPlayerListener(this);
mediaPlayer.realize();
mediaPlayer.prefetch();
After start I to use mediaPlayer.getDuration it returns lets say around 24:22 (the inbuild media player in the blackberry say the file length is 4:05)
I tried to get the duration in the listener and there it unfortunately returned around 64 minutes, so I'm sure something is not good inside the datasoruce....
code that converts
String getElapsedTimeMinutesSeconds(long elapsedTime) {
long Seconds=(elapsedTime/1000)%60;
long Minutes=(elapsedTime/(1000*60))%60;
long Hours=(elapsedTime/(1000*60*60))%24;
return ""+Minutes + ":"+Seconds;
}
Player.setMediaTime() and Player.getMediaTime() both refer to time in microseconds, not milliseconds. So to get the number of elapsed seconds, you need to divide by 1000000 instead of just 1000.
Sometime earlier I faced with the problem to play unlimited size audio. I fix it.
Here's the link:
http://supportforums.blackberry.com/t5/Java-Development/Use-custom-DataSource-to-play-audio/m-p/1373247#M178928

Resources