I am working on smart contract for a simple payment system where u deposit and get a code. Then u can give a code to anyone and they can withdraw to any address.
I have already tested on all testnets and it works fine.
But on ethereum mainnet as soon as I deposit the eth, it is stolen and transferred by a mev bot.
//SPDX-License-Identifier: UNLICENSED
// It will be used by the Solidity compiler to validate its version.
pragma solidity ^0.8.9;
// We import this library to be able to use console.log
import "hardhat/console.sol";
import "#openzeppelin/contracts/access/Ownable.sol";
contract Payment is Ownable {
address payable public feeWallet;
uint256 public feeAmount;
uint256 private numberOfDeposits = 0;
struct Deposit {
uint256 time;
bytes32 id;
uint256 amount;
}
// mapping(address => bytes32[]) public addressToHashes;
mapping(bytes32 => uint256) public hashToAmount;
mapping(address => Deposit[]) public addressToDeposits;
mapping(bytes32 => bool) public isWithdrawn;
event DepositCreated(address _payer, uint256 _amount);
event DepositWithdrawn(
address _withdrawer,
uint256 _amount,
address indexed _withdrawTo
);
constructor(address _feeWallet) {
feeWallet = payable(_feeWallet);
}
function createDeposit(string memory _id) public payable returns (bytes32) {
//Check if value is greater than 0
require(msg.value > 0, "Amount cannot be equal to 0");
//Fetch mapppings
Deposit[] storage deposits = addressToDeposits[msg.sender];
//Create Hash from id
bytes32 hash = generateId(_id);
//Update Mappings
hashToAmount[hash] = msg.value;
numberOfDeposits += 1;
deposits.push(Deposit(block.timestamp, hash, msg.value));
addressToDeposits[msg.sender] = deposits;
emit DepositCreated(msg.sender, msg.value);
return hash;
}
function generateId(string memory _id) internal view returns (bytes32) {
uint id = uint(
keccak256(
abi.encodePacked(
_id,
block.difficulty,
block.timestamp,
numberOfDeposits
)
)
);
return bytes32(id % 100000000000000);
}
function checkDepositExist(uint _id) public view returns (bool) {
bytes32 hash = bytes32(_id);
uint256 amount = hashToAmount[hash];
console.log("amount", amount);
if (amount > 0) {
return true;
} else {
return false;
}
}
function viewDeposit(uint _id) public view returns (uint256) {
bytes32 hash = bytes32(_id);
uint256 amount = hashToAmount[hash];
return amount;
}
function withdrawDeposit(uint _id, address _to) public {
require(checkDepositExist(_id), "ID invalid");
bytes32 hash = bytes32(_id);
uint256 amount = hashToAmount[hash] - feeAmount;
hashToAmount[hash] = 0;
(bool sent, bytes memory data) = payable(_to).call{value: amount}("");
require(sent, "Failed to send deposit amount");
require(sendFees(), "Failed to send fee amount");
isWithdrawn[hash] = true;
emit DepositWithdrawn(msg.sender, amount, _to);
}
function getUserDeposits() public view returns (Deposit[] memory) {
Deposit[] memory deposits = addressToDeposits[msg.sender];
return deposits;
}
function setFeeAmount(uint _amount) public onlyOwner {
feeAmount = _amount;
}
function setFeeWallet(address _wallet) public onlyOwner {
feeWallet = payable(_wallet);
}
function sendFees() internal returns (bool) {
(bool sent, bytes memory data) = payable(feeWallet).call{
value: feeAmount
}("");
return sent;
}
}
Here's the transaction of eth being stolen https://etherscan.io/tx/0xd4f92a3346ff51cf41c40b47b1270eda5ca57c4aaae2b3c9858298d8c6269725
I can point out many vulnerabilities that this smart contract has.
One of them is that there is no private data in a smart contract. Anybody can read the data in the storage of a smart contract. In this case, I see that hashToAmount has the hashes that you send to the user who created the deposit. But this can be read by anyone (using something like web3.eth.getStorageAt(contractAddress, storageIndex)) and call the withdrawDeposit function and steal the funds.
Something else, miners can temper with some blockchain data, like the timestamp, etc. Miners can see the _id that is being sent to the createDeposit and use it to immediately steal the funds from the contract.
Also, I see a lack of checks in the withdrawDeposit function.
I suggest that you use well known patterns and modifiers and other checks. Relying more on things like msg.sender since nobody can fake or tamper with msg.sender value. The sender is always the user that created the request, and if the sender is the owner of some ether, than only they can withdraw it.
Instead of checking the balance of that hash, save the msg.sender in the mapping instead when someone makes a deposit. Then, while withdrawing, check that the msg.sender has balance and if so, send the balance to the _to address.
Also, check for reentrancy attacks in your withdrawDeposit deposit function.
Trying to generate an id in a smart contract with the block data is not that secure. There are oracle libraries that can help you get a pseudo-random number from outside of the contract, but again, it can be read by a miner before the transaction is included in a block and compromise your funds.
I'm not sure if any of these are causing your problem exactly, but I'm sure that all of these could be potential issues for your contract.
I recommend you follow security standard patterns like from OpenZeppelin: https://www.openzeppelin.com/.
I am following along full blockchain solidity course.
I am trying to deploy fund_and_withdraw.py to Rinkeby using brownie but I run into the follwoing error:
ValueError: Gas estimation failed: 'execution reverted: You need to spend more ETH!'. This transaction will likely revert. If you wish to broadcast, you must set the gas limit manually.
I previously tried deploying it on ganache-cli but get a "Index out of range" issue as described on my previous issue.
Now, I am trying to run it on Rinkeby (Ganache UI is not connecting to my Brownie and is throwing off the entire contract) but it returns the "ValueError".
Also, tried changing the decimals in FundMe.py.
I tried deploying on Rinkby and I no longer get "Index out of range" but instead i get "ValueError"
My code below:
fund_and_withdraw
from brownie import FundMe
from scripts.helpful_scripts import get_account
def fund():
fund_me = FundMe[-1]
account = get_account()
entrance_fee = fund_me.getEntranceFee()
print(entrance_fee)
print(f"The current entry fee is {entrance_fee}")
print("funding")
fund_me.fund(
{
"from": account,
"value": entrance_fee,
}
)
# 0.025000000000000000
def main():
fund()
Helpful_scripts.py
from brownie import network, config, accounts, MockV3Aggregator
from web3 import Web3
# Create Variable to store dev network list (Ex: ["development", "ganache", etc...])
LOCAL_BLOCKCHAIN_ENVIRONMENTS = "development"
DECIMALS = 8
STARTING_PRICE = 200000000000
def get_account():
# if network.show_active == "development":
if network.show_active() in LOCAL_BLOCKCHAIN_ENVIRONMENTS:
return accounts[0]
else:
return accounts.add(config["wallets"]["from_key"])
def deploy_mocks():
print(f"The active network is {network.show_active()}")
print("Deploying Mocks...")
if len(MockV3Aggregator) <= 0:
MockV3Aggregator.deploy(
# DECIMALS, Web3.toWei(STARTING_PRICE, "ether"), {"from": get_account()}
DECIMALS,
STARTING_PRICE,
{"from": get_account()},
)
print("Mocks Deployed!")
FundMe.py
contract FundMe {
using SafeMath96 for uint256;
mapping(address => uint256) public addressToAmountFunded;
address[] public funders;
address public owner;
AggregatorV3Interface public priceFeed;
//constructor(address _priceFeed) public {
constructor(address _priceFeed) {
priceFeed = AggregatorV3Interface(_priceFeed);
owner = msg.sender;
}
function fund() public payable {
uint256 minimumUSD = 50 * 10**18;
require(
getConversionRate(msg.value) >= minimumUSD,
"You need to spend more ETH!"
);
addressToAmountFunded[msg.sender] += msg.value;
funders.push(msg.sender);
}
function getVersion() public view returns (uint256) {
return priceFeed.version();
}
function getPrice() public view returns (uint256) {
(, int256 answer, , , ) = priceFeed.latestRoundData();
//return uint256(answer * 10000000000);
return uint256(answer * 100000000);
}
// 1000000000
function getConversionRate(uint256 ethAmount)
public
view
returns (uint256)
{
uint256 ethPrice = getPrice();
//uint256 ethAmountInUsd = (ethPrice * ethAmount) / 1000000000000000000;
uint256 ethAmountInUsd = (ethPrice * ethAmount) / 100000000;
return ethAmountInUsd;
}
function getEntranceFee() public view returns (uint256) {
// minimumUSD
uint256 minimumUSD = 50 * 10**18;
uint256 price = getPrice();
uint256 precision = 1 * 10**18;
// return (minimumUSD * precision) / price;
// We fixed a rounding error found in the video by adding one!
return ((minimumUSD * precision) / price) + 1;
}
modifier onlyOwner() {
require(msg.sender == owner);
_;
}
function withdraw() public payable onlyOwner {
payable(msg.sender).transfer(address(this).balance);
for (
uint256 funderIndex = 0;
funderIndex < funders.length;
funderIndex++
) {
address funder = funders[funderIndex];
addressToAmountFunded[funder] = 0;
}
funders = new address[](0);
}
}
Thank you!
Your require statement is failing in this line:
require(
getConversionRate(msg.value) >= minimumUSD,
"You need to spend more ETH!"
);
This probably means that you don't have enough ether in your wallet to send to the contract. So fund your wallet with more ether and it will probably work if you deployed the contract correctly.
The index out of range is probably a fault in your get_account() function it seems. The LOCAL_BLOCKCHAIN_ENVIRONMENTS should be an array not a string. It also doesn't contain the ganache-local so you are trying to use your private key for your local blockchain that is probably why you are getting index out of range.
So fix your LOCAL_BLOCKCHAIN_ENVIRONMENTS to look like this:
LOCAL_BLOCKCHAIN_ENVIRONMENTS = ["development", "ganache-local"]
I guess it should work then.
from brownie import FundMe
from scripts.helpful_scripts import get_account
def fund():
_priceFeed = 0x8A753747A1Fa494EC906cE90E9f37563A8AF630e
account = get_account()
fund_me = FundMe.deploy(_priceFeed , {"from": account})
**_priceFeed is in your constructor , so you should pass it to your
contract for deploy, i added it as hardcode , but its better to use yaml
file for projects **
entrance_fee = fund_me.getEntranceFee()
print(entrance_fee)
print(f"The current entry fee is {entrance_fee}")
print("funding")
fund_tx = fund_me.fund(
{
"from": account,
"value": entrance_fee,
}
)
# 0.025000000000000000
fund_tx.wait(1)
** when you use functions that make transaction(not reading) , its better
to use wait() method , its help your transaction to write in block **
def main():
fund()
I can not verify my test smart contract on BSC Main Network using #chainlink.
// SPDX-License-Identifier: MIT
pragma solidity ^0.6.7;
import "#chainlink/contracts/src/v0.6/interfaces/AggregatorV3Interface.sol";
contract PriceTest {
AggregatorV3Interface internal priceFeed;
constructor() public {
priceFeed = AggregatorV3Interface(0x0567F2323251f0Aab15c8dFb1967E4e8A7D42aeE); // for BSC Main Net
}
function getLatestPrice() public view returns (uint256) {
(,int price,,,) = priceFeed.latestRoundData();
return uint256(price/100000000);
}
}
What I have done :
Make Flattened file and put in the contract code using truffle-flattener. I did it before for a test net and it's ok.
Copy ABI from remix and pasted to abi.hashex.org
But I tried many time end up with problem in verifying the contract. I think I am not really understand how to put constructor parameter correctly to get the ABI auto-parse.
I am quite new to smart contract. Need learn more from experts.
My contract :
https://bscscan.com/address/0x0849a15338a5f0787696cea335757b608ff92a85
Flattened file :
// File: #chainlink\contracts\src\v0.6\interfaces\AggregatorV3Interface.sol
// SPDX-License-Identifier: MIT
pragma solidity ^0.6.0;
interface AggregatorV3Interface {
function decimals()
external
view
returns (
uint8
);
function description()
external
view
returns (
string memory
);
function version()
external
view
returns (
uint256
);
// getRoundData and latestRoundData should both raise "No data present"
// if they do not have data to report, instead of returning unset values
// which could be misinterpreted as actual reported values.
function getRoundData(
uint80 _roundId
)
external
view
returns (
uint80 roundId,
int256 answer,
uint256 startedAt,
uint256 updatedAt,
uint80 answeredInRound
);
function latestRoundData()
external
view
returns (
uint80 roundId,
int256 answer,
uint256 startedAt,
uint256 updatedAt,
uint80 answeredInRound
);
}
// File: contracts\Getprice.sol
pragma solidity ^0.6.7;
contract PriceTest {
AggregatorV3Interface internal priceFeed;
constructor() public {
priceFeed = AggregatorV3Interface(0x0567F2323251f0Aab15c8dFb1967E4e8A7D42aeE); // for BSC Main Net
}
function getLatestPrice() public view returns (uint256) {
(,int price,,,) = priceFeed.latestRoundData();
return uint256(price/100000000);
}
}
I have been trying to learn how to make a GET request using Chainlink.
I was watching this tutorial: https://www.youtube.com/watch?v=ay4rXZhAefs and I used the exact same code that they used in the video and I also followed the same steps using Remix (I have my metamask (Kovan) open with enough ETH and LINK and I also funded my contract with enough LINK). However after I call requestVolumeData() the volume variable doesn't update and stays 0.
I have tried to use different networks and oracles/jobs, but I seem to run into the same issue over and over again. Any help would be appreciated.
pragma solidity ^0.6.0;
import "#chainlink/contracts/src/v0.6/ChainlinkClient.sol";
/**
* THIS IS AN EXAMPLE CONTRACT WHICH USES HARDCODED VALUES FOR CLARITY.
* PLEASE DO NOT USE THIS CODE IN PRODUCTION.
*/
contract APIConsumer is ChainlinkClient {
using Chainlink for Chainlink.Request;
uint256 public volume;
address private oracle;
bytes32 private jobId;
uint256 private fee;
/**
* Network: Kovan
* Oracle: 0x2f90A6D021db21e1B2A077c5a37B3C7E75D15b7e
* Job ID: 29fa9aa13bf1468788b7cc4a500a45b8
* Fee: 0.1 LINK
*/
constructor() public {
setPublicChainlinkToken();
oracle = 0x2f90A6D021db21e1B2A077c5a37B3C7E75D15b7e;
jobId = "29fa9aa13bf1468788b7cc4a500a45b8";
fee = 0.1 * 10 ** 18; // (Varies by network and job)
}
/**
* Create a Chainlink request to retrieve API response, find the target
* data, then multiply by 1000000000000000000 (to remove decimal places from data).
*/
function requestVolumeData() public returns (bytes32 requestId)
{
Chainlink.Request memory request = buildChainlinkRequest(jobId, address(this), this.fulfill.selector);
// Set the URL to perform the GET request on
request.add("get", "https://min-api.cryptocompare.com/data/pricemultifull?fsyms=ETH&tsyms=USD");
// Set the path to find the desired data in the API response, where the response format is:
// {"RAW":
// {"ETH":
// {"USD":
// {
// "VOLUME24HOUR": xxx.xxx,
// }
// }
// }
// }
request.add("path", "RAW.ETH.USD.VOLUME24HOUR");
// Multiply the result by 1000000000000000000 to remove decimals
int timesAmount = 10**18;
request.addInt("times", timesAmount);
// Sends the request
return sendChainlinkRequestTo(oracle, request, fee);
}
/**
* Receive the response in the form of uint256
*/
function fulfill(bytes32 _requestId, uint256 _volume) public recordChainlinkFulfillment(_requestId)
{
volume = _volume;
}
// function withdrawLink() external {} - Implement a withdraw function to avoid locking your LINK in the contract
}
Are you giving some time to let the node fetch the request? When you do this example, you usually have to click the requestVolumeData() button, multiple times, waiting for the node to fulfill the request. Less than two minutes usually.
Here's another more recent version of that example contract. It's nearly the same.
// SPDX-License-Identifier: MIT
pragma solidity ^0.6.6;
import "#chainlink/contracts/src/v0.6/ChainlinkClient.sol";
contract APIConsumer is ChainlinkClient {
uint256 public volume;
address private oracle;
bytes32 private jobId;
uint256 private fee;
/**
* Network: Kovan
* Oracle: 0x2f90A6D021db21e1B2A077c5a37B3C7E75D15b7e
* Job ID: 29fa9aa13bf1468788b7cc4a500a45b8
* Fee: 0.1 LINK
*/
constructor(address _oracle, string memory _jobId, uint256 _fee, address _link) public {
if (_link == address(0)) {
setPublicChainlinkToken();
} else {
setChainlinkToken(_link);
}
// oracle = 0x2f90A6D021db21e1B2A077c5a37B3C7E75D15b7e;
// jobId = "29fa9aa13bf1468788b7cc4a500a45b8";
// fee = 0.1 * 10 ** 18; // 0.1 LINK
oracle = _oracle;
jobId = stringToBytes32(_jobId);
fee = _fee;
}
/**
* Create a Chainlink request to retrieve API response, find the target
* data, then multiply by 1000000000000000000 (to remove decimal places from data).
*/
function requestVolumeData() public returns (bytes32 requestId)
{
Chainlink.Request memory request = buildChainlinkRequest(jobId, address(this), this.fulfill.selector);
// Set the URL to perform the GET request on
request.add("get", "https://min-api.cryptocompare.com/data/pricemultifull?fsyms=ETH&tsyms=USD");
// Set the path to find the desired data in the API response, where the response format is:
// {"RAW":
// {"ETH":
// {"USD":
// {
// "VOLUME24HOUR": xxx.xxx,
// }
// }
// }
// }
request.add("path", "RAW.ETH.USD.VOLUME24HOUR");
// Multiply the result by 1000000000000000000 to remove decimals
int timesAmount = 10**18;
request.addInt("times", timesAmount);
// Sends the request
return sendChainlinkRequestTo(oracle, request, fee);
}
/**
* Receive the response in the form of uint256
*/
function fulfill(bytes32 _requestId, uint256 _volume) public recordChainlinkFulfillment(_requestId)
{
volume = _volume;
}
function stringToBytes32(string memory source) public pure returns (bytes32 result) {
bytes memory tempEmptyStringTest = bytes(source);
if (tempEmptyStringTest.length == 0) {
return 0x0;
}
assembly {
result := mload(add(source, 32))
}
}
}
The page about the PubNub History API states that
The history() function returns a list of up to 100 messages, the start
time token and the ending time token.
Is there a way to retrieve more than the 100 messages?
I'm currently not a paying customer of PubNub.
PubNub Load History More than 100 Messages
Sometimes you want to slice back in time over a linear stream of data. And often you'll want to do this at different levels of granularity. That is why PubNub Storage and Playback APIs provide maximum level of flexibility. However sometimes it ends up being a bit tricky to load data with the preferred result set.
PubNub Real-Time Network Storage and Playback
There are several considerations you may be seeking when loading transaction history over timelines that can potentially span millions of message in the transaction set. There are some great options available to you and we will cover two of them right now. The examples will be coded in JavaScript. The first example loads a summary of the data by grabbing the snapshots for the beginning of each hour for the past 24 hours. The second example shows you how to load all transactions in full detail and maximum granularity.
All Reference Files can be found on this GIST: Loading History from PubNub Mt.Gox Trades
Example PubNub Mt.Gox History JavaScript Usage
<script src="https://cdn.pubnub.com/pubnub.min.js"></script>
<script src="mtgox-history.js"></script>
<script>(function(){
// LOAD HOURLY SUMMARY
MTGOX.history.hourly({
channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f',
data : function(response) { console.log(JSON.stringify(response)) },
error : function() { console.log("NETWORK ERROR") }
});
// LOAD ALL WITH LIMITER OPTION
MTGOX.history.full({
limit : 500, // SET LIMIT AS HIGH AS NEEDED TO LOAD MORE!
channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f',
data : function(messages) { console.log(messages) },
error : function(e) { console.log("NETWORK ERROR") }
});
})();</script>
NOTE: Running MTGOX.history.hourly() method will generate a list of snapshots per hour over the last 24 hours.
NOTE: Running MTGOX.history.full() method will generate maximum resolution detail with a lot of data. You can get a full dump or partial dump as needed; and you should increase the limit parameter in order to grab more data points.
This following JavaScript file will provide you the MTGOX interface.
PubNub Mt.Gox History JavaScript Loader
//
// mtgox-history.js
//
(function(){
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// INITIALIZE PUBNUB
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
var pubnub = PUBNUB.init({
subscribe_key : 'sub-c-50d56e1e-2fd9-11e3-a041-02ee2ddab7fe'
});
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// MTGOX HISTORY INTERFACE
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
window.MTGOX = {
history : {
hourly : hourly,
full : full
}
};
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// GET ALL DATA FOREVER (WITH LIMIT OF COURSE)
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
/*
MTGOX.history.full({
limit : 1000,
channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f',
data : function(messages) { console.log(messages) },
error : function(e) { console.log("NETWORK ERROR") }
});
*/
function full(args) {
var chan = args['channel'] ||'d5f06780-30a8-4a48-a2f8-7ed181b4a13f'
, callback = args['data'] || function(){}
, error = args['error'] || function(){}
, limit = +args['limit'] || 5000
, start = 0
, count = 100
, history = []
, params = {
channel : chan,
count : count,
callback : function(messages) {
var msgs = messages[0];
start = messages[1];
params.start = start;
PUBNUB.each( msgs.reverse(), function(m) {history.push(m)} );
if (history.length >= limit) return callback(history);
if (msgs.length < count) return callback(history);
count = 100;
add_messages();
},
error : function(e) {
callback(history);
error(history);
}
};
add_messages();
function add_messages() { pubnub.history(params) }
}
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// GET 24 HOURS IN HOURLY INCREMENTS
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
/*
MTGOX.history.hourly({
channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f',
data : function(response) { console.log(response) },
error : function() { console.log('ERROR') }
});
*/
function hourly(setup) {
var limit = 24;
var count = 0;
var chan = setup['channel'] ||'d5f06780-30a8-4a48-a2f8-7ed181b4a13f';
var cb = setup['data'] || function(){};
var eb = setup['error'] || function(){};
var now = new Date();
now.setUTCHours(0);
now.setUTCMinutes(0);
now.setUTCSeconds(0);
now.setUTCMilliseconds(0);
var utc_now = now.getTime();
var vectors = [];
PUBNUB.each( (new Array(limit)).join(',').split(','), function( _, d ) {
var day = utc_now - 3600000 * d;
pubnub.history({
limit : 1,
channel : chan,
start : day * 10000,
error : function() { count++; eb(); },
callback : function(messages) {
// DONE?
if (++count == limit) return cb(vectors);
// ADD TIME SLICES
var res = +(((messages[0][0]||{}).ticker||{}).avg||{}).value;
res && vectors.push([ new Date(day).getUTCHours(), res ]);
// KEEP IT SORTED
vectors.sort(function(a,b){ return a[0] > b[0] && -1 || 1 });
}
})
} );
}
})();
Mt.Gox PubNub Channel Listing for Tickers, Depth and Trades
The following is a list of channels provided by Mt.Gox data feed options you can use in the history channel parameter field.
{
"TICKER.ltcgbp": "0102A446-E4D4-4082-8E83-CC02822F9172",
"TICKER.ltccny": "0290378C-E3D7-4836-8CB1-2BFAE20CC492",
"DEPTH.btchkd": "049F65DC-3AF3-4FFD-85A5-AAC102B2A579",
"DEPTH.btceur": "057BDC6B-9F9C-44E4-BC1A-363E4443CE87",
"TICKER.nmcaud": "08C65460-CBD9-492E-8473-8507DFA66AE6",
"TICKER.btceur": "0BB6DA8B-F6C6-4ECF-8F0D-A544AD948C15",
"DEPTH.btckrw": "0C84BDA7-E613-4B19-AE2A-6D26412C9F70",
"DEPTH.btccny": "0D1ECAD8-E20F-459E-8BED-0BDCF927820F",
"TICKER.btccad": "10720792-084D-45BA-92E3-CF44D9477775",
"DEPTH.btcchf": "113FEC5F-294D-4929-86EB-8CA4C3FD1BED",
"TICKER.ltcnok": "13616AE8-9268-4A43-BDF7-6B8D1AC814A2",
"TICKER.ltcusd": "1366A9F3-92EB-4C6C-9CCC-492A959ECA94",
"TICKER.btcbtc": "13EDFF67-CFA0-4D99-AA76-52BD15D6A058",
"TICKER.ltccad": "18B55737-3F5C-4583-AF63-6EB3951EAD72",
"TICKER.nmccny": "249FDEFD-C6EB-4802-9F54-064BC83908AA",
"DEPTH.btcusd": "24E67E0D-1CAD-4CC0-9E7A-F8523EF460FE",
"TICKER.btcchf": "2644C164-3DB7-4475-8B45-C7042EFE3413",
"DEPTH.btcaud": "296EE352-DD5D-46F3-9BEA-5E39DEDE2005",
"TICKER.btcczk": "2A968B7F-6638-40BA-95E7-7284B3196D52",
"TICKER.btcsgd": "2CB73ED1-07F4-45E0-8918-BCBFDA658912",
"TICKER.nmcjpy": "314E2B7A-A9FA-4249-BC46-B7F662ECBC3A",
"TICKER.btcnmc": "36189B8C-CFFA-40D2-B205-FB71420387AE",
"DEPTH.btcinr": "414FDB18-8F70-471C-A9DF-B3C2740727EA",
"DEPTH.btcsgd": "41E5C243-3D44-4FAD-B690-F39E1DBB86A8",
"TICKER.btcltc": "48B6886F-49C0-4614-B647-BA5369B449A9",
"TICKER.ltceur": "491BC9BB-7CD8-4719-A9E8-16DAD802FFAC",
"TICKER.btcinr": "55E5FEB8-FEA5-416B-88FA-40211541DECA",
"TICKER.ltcjpy": "5AD8E40F-6DF3-489F-9CF1-AF28426A50CF",
"DEPTH.btccad": "5B234CC3-A7C1-47CE-854F-27AEE4CDBDA5",
"TICKER.btcnzd": "5DDD27CA-2466-4D1A-8961-615DEDB68BF1",
"DEPTH.btcgbp": "60C3AF1B-5D40-4D0E-B9FC-CCAB433D2E9C",
"DEPTH.btcnok": "66DA7FB4-6B0C-4A10-9CB7-E2944E046EB5",
"DEPTH.btcthb": "67879668-532F-41F9-8EB0-55E7593A5AB8",
"TICKER.btcsek": "6CAF1244-655B-460F-BEAF-5C56D1F4BEA7",
"TICKER.btcnok": "7532E866-3A03-4514-A4B1-6F86E3A8DC11",
"TICKER.btcgbp": "7B842B7D-D1F9-46FA-A49C-C12F1AD5A533",
"TRADE.LAG": "85174711-BE64-4DE1-B783-0628995D7914",
"DEPTH.btcsek": "8F1FEFAA-7C55-4420-ADA0-4DE15C1C38F3",
"DEPTH.btcdkk": "9219ABB0-B50C-4007-B4D2-51D1711AB19C",
"DEPTH.btcjpy": "94483E07-D797-4DD4-BC72-DC98F1FD39E3",
"TICKER.nmcusd": "9AAEFD15-D101-49F3-A2FD-6B63B85B6BED",
"TICKER.ltcaud": "A046600A-A06C-4EBF-9FFB-BDC8157227E8",
"TICKER.btcjpy": "A39AE532-6A3C-4835-AF8C-DDA54CB4874E",
"DEPTH.btcczk": "A7A970CF-4F6C-4D85-A74E-AC0979049B87",
"TICKER.ltcdkk": "B10A706E-E8C7-4EA8-9148-669F86930B36",
"TICKER.btcpln": "B4A02CB3-2E2D-4A88-AEEA-3C66CB604D01",
"TEST": "BAD99F24-FA8B-4938-BFDF-0C1831FC6665",
"TICKER.btcrub": "BD04F720-3C70-4DCE-AE71-2422AB862C65",
"TICKER.nmcgbp": "BF5126BA-5187-456F-8AE6-963678D0607F",
"TICKER.btckrw": "BF85048D-4DB9-4DBE-9CA3-5B83A1A4186E",
"TICKER.btccny": "C251EC35-56F9-40AB-A4F6-13325C349DE4",
"DEPTH.btcnzd": "CEDF8730-BCE6-4278-B6FE-9BEE42930E95",
"TICKER.btchkd": "D3AE78DD-01DD-4074-88A7-B8AA03CD28DD",
"TICKER.btcthb": "D58E3B69-9560-4B9E-8C58-B5C0F3FDA5E1",
"TICKER.btcusd": "D5F06780-30A8-4A48-A2F8-7ED181B4A13F",
"DEPTH.btcrub": "D6412CA0-B686-464C-891A-D1BA3943F3C6",
"TICKER.nmceur": "D8512D04-F262-4A14-82F2-8E5C96C15E68",
"TRADE.btc": "DBF1DEE9-4F2E-4A08-8CB7-748919A71B21",
"TICKER.nmccad": "DC28033E-7506-484C-905D-1C811A613323",
"DEPTH.btcpln": "E4FF055A-F8BF-407E-AF76-676CAD319A21",
"TICKER.btcdkk": "E5CE0604-574A-4059-9493-80AF46C776B3",
"TICKER.btcaud": "EB6AAA11-99D0-4F64-9E8C-1140872A423D"
}
See https://help.pubnub.com/entries/24113341-How-do-I-Page-Through-Stored-Messages-
Contact PubNub support (help#...) if further assistance is needed
The below Java class can be used to easily retrieve and process long time ranges of history messages in an advanced for loop of this form:
PubnubHistoryExcerpt history = new PubnubHistoryExcerpt(pubnub, channel, start, end);
for (Object message : history) {
// do something with the message object
}
The messages are retrieved on the fly, so no memory problem occurs.
Below is the complete code. You can find a fully runnable usage example in the main() method inside of the class.
I haven't yet tested this class extensively. Enhancements are welcome.
/*
* PubnubHistoryExcerpt.java
*
* This file is distributed under the FreeBSD License:
*
* Copyright (c) 2014, Daniel S. (http://stackoverflow.com/users/1838726/daniel-s)
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* The views and conclusions contained in the software and documentation are those
* of the authors and should not be interpreted as representing official policies,
* either expressed or implied, of the FreeBSD Project.
*/
import java.math.BigDecimal;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.NoSuchElementException;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.pubnub.api.Callback;
import com.pubnub.api.Pubnub;
import com.pubnub.api.PubnubError;
/**
* You can use this class to iterate over historical PubNub messages. The messages are retrieved transparently while you
* iterate over the history excerpt. This class and the returned iterators are thread-safe.
*
* See {#link #main(String[])} for a usage example.
*/
public class PubnubHistoryExcerpt implements Iterable<Object> {
/**
* This main method contains a usage example for this class. It downloads last 3 hour's messages of the MtGox BTC/USD ticker
* channel from PubNub and outputs the timestamp and USD value which are found in the messages.
*/
public static void main(String[] args) throws JSONException {
String PUBNUB_SUBSCRIBE_KEY_MTGOX = "sub-c-50d56e1e-2fd9-11e3-a041-02ee2ddab7fe";
String PUBNUB_CHANNEL_MTGOX_TICKER_BTCUSD = "d5f06780-30a8-4a48-a2f8-7ed181b4a13f";
Pubnub pubnub = new Pubnub(null, PUBNUB_SUBSCRIBE_KEY_MTGOX);
long ONE_HOUR_IN_MILLIS = 60 * 60 * 1000;
long end = System.currentTimeMillis();
long start = end - 3 * ONE_HOUR_IN_MILLIS;
// convert from milliseconds as time unit (10^-3 seconds) to
// pubnub's better-than-microsecond precision time units (10^-7 seconds)
start *= 10000;
end *= 10000;
PubnubHistoryExcerpt history = new PubnubHistoryExcerpt(pubnub, PUBNUB_CHANNEL_MTGOX_TICKER_BTCUSD, start, end);
DefaultDateFormat dateFormat = DefaultDateFormat.create();
for (Object message : history) {
JSONObject messageJson = (JSONObject) message;
JSONObject ticker = messageJson.getJSONObject("ticker");
long instant = ticker.getLong("now");
BigDecimal value = new BigDecimal(ticker.getJSONObject("last_local").getString("value"));
instant /= 1000; // convert from microseconds to milliseconds
System.out.println(dateFormat.format(instant) + ": " + value);
}
System.exit(0);
}
/**
* This is the maximum number of messages to fetch in one batch. If you fetch many messages, higher numbers improve
* performance. Setting this to a value higher than 100 doesn't have an effect, because Pubnub currently doesn't
* support fetching more than this many messages at once.
*/
private static final int BATCH_SIZE = 100;
private final Pubnub pubnub;
private final String channel;
private final long start;
private final long end;
/**
* Constructs a new excerpt over which you can iterate. Insances represent an excerpt. No retrieval operations are
* started unless you call iterator().next() for the first time.
*
* #param pubnub
* The Pubnub connection to use for retrieving messages.
* #param channel
* The channel for which to retrieve historical messages.
* #param start
* The beginning of the time interval for which to retrieve messages, in pubnub's time units (10^-7
* seconds, so milliseconds * 10000) since 1970-01-01 00:00:00).
* #param end
* The end of the time interval for which to retrieve messages, in pubnub's time units (10^-7 seconds, so
* milliseconds * 10000) since 1970-01-01 00:00:00).
*/
private PubnubHistoryExcerpt(Pubnub pubnub, String channel, long start, long end) {
this.pubnub = pubnub;
this.channel = channel;
this.start = start;
this.end = end;
}
public Iterator<Object> iterator() {
return new Iter();
}
private class Iter implements Iterator<Object> {
/**
* This list is used as a fifo buffer for messages retrieves through this iterator. It also acts as the main
* synchronization lock for synchronizing access between threads accessing this class as an iterator and threads
* calling back from the Pubnub API.
*/
private LinkedList<Object> buffer = new LinkedList<Object>();
/**
* This field stores the end of the time range of the previous batch retrieval, in Pubnub time units (10th of a
* microsecond, so milliseconds*10000). For the following batch retrieval, this is used as the start time for
* retrieving the following messages.
*/
private long prevBatchTimeRangeEnd = PubnubHistoryExcerpt.this.start;
/**
* Retrieval of messages is handled asynchronously. That means that exceptions which are thrown during retrieval
* can't automatically be propagated through to the code which invokes <code>next()</code> or
* <code>hasNext()</code> . Therefor, such an exception is stored temporarily in this field and then re-thrown
* from within <code>next()</code> or <code>hasNext()</code>.
*/
private Exception caughtDuringRetrieval = null;
/**
* This object is used to wait on and to notify about updates of the buffer.
*/
private Object notifier = new Object();
/**
* Because of spurious wakeups that can happen during wait(), this field is necessary to tell the waiting thread
* if retrieval is still running.
*/
private boolean retrieving = false;
/**
* The callback object to use for retrieving messages. This is stored in a field here for re-use. This is a
* compromise between performance and low memory footprint, slightly in favor of performance.
*/
private InternalCallback internalCallback = new InternalCallback();
private void retrieveNextBatch() {
synchronized (notifier) {
this.retrieving = true;
// String startStr = DefaultDateFormat.create().format(prevBatchTimeRangeEnd / 10000);
// String endStr = DefaultDateFormat.create().format(end / 10000);
// System.out.println("fetching from " + startStr + " till " + endStr);
if (Iter.this.prevBatchTimeRangeEnd < PubnubHistoryExcerpt.this.end) {
PubnubHistoryExcerpt.this.pubnub.history( //
PubnubHistoryExcerpt.this.channel, //
Iter.this.prevBatchTimeRangeEnd, //
PubnubHistoryExcerpt.this.end, //
BATCH_SIZE, //
false, //
Iter.this.internalCallback //
);
waitUntilNextBatchRetrievalFinished();
}
}
}
private void waitUntilNextBatchRetrievalFinished() {
while (this.retrieving) {
try {
this.notifier.wait();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
private class InternalCallback extends Callback {
#Override
public void successCallback(String channel, Object message) {
synchronized (Iter.this.notifier) {
try {
processSuccessCallback(channel, message);
} catch (Exception e) {
Iter.this.caughtDuringRetrieval = e;
} finally {
Iter.this.retrieving = false;
Iter.this.notifier.notifyAll();
}
}
}
#Override
public void errorCallback(String channel, PubnubError error) {
Iter.this.caughtDuringRetrieval = new Exception("" + //
error.getClass().getName() + ": " + //
error.getErrorString() + //
" (code=" + error.errorCode + "; extendedCode=" + error.errorCodeExtended + ")");
Iter.this.caughtDuringRetrieval.fillInStackTrace();
}
}
private void processSuccessCallback(String channel, Object message) throws JSONException {
if (message == null)
throw new NullPointerException("retrieved message is null");
if (!(message instanceof JSONArray))
throw new RuntimeException("retrieved message is not a " + JSONArray.class.getName());
JSONArray historyMessage = (JSONArray) message;
// System.out.println(historyMessage.toString(2));
JSONArray messageList = extractMessageList(historyMessage);
long batchTimeRangeEnd = extractBatchTimeRangeEnd(historyMessage);
if (batchTimeRangeEnd > 0)
Iter.this.prevBatchTimeRangeEnd = batchTimeRangeEnd;
else
Iter.this.prevBatchTimeRangeEnd = end;
processMessageList(messageList);
}
private void processMessageList(JSONArray messageList) {
int i = 0;
for (; i < messageList.length(); i++) {
JSONObject message;
try {
message = messageList.getJSONObject(i);
} catch (JSONException e) {
String str;
try {
str = messageList.toString(2);
} catch (JSONException secondaryE) {
str = "(couldn't convert messageList to String because of " + secondaryE.toString() + ")";
}
throw new RuntimeException("couldn't extract message at index " + i + " from messageList (messageList:\n" + str
+ "\n(end of messageList)\n)", e);
}
Iter.this.buffer.add(message);
}
}
private long extractBatchTimeRangeEnd(JSONArray historyMessage) {
long batchTimeRangeEnd;
try {
batchTimeRangeEnd = historyMessage.getLong(2);
} catch (JSONException e) {
String str = safeConvertHistoryMessageToString(historyMessage);
throw new RuntimeException("could not extract element 2 (batchTimeRangeEnd) of retrieved historyMessage (historyMessage:\n" + str
+ "\n(end of historyMessage)\n)", e);
}
return batchTimeRangeEnd;
}
private String safeConvertHistoryMessageToString(JSONArray historyMessage) {
String str;
try {
str = historyMessage.toString(2);
} catch (JSONException secondaryE) {
str = "(couldn't convert historyMessage to String because of " + secondaryE.toString() + ")";
}
return str;
}
private JSONArray extractMessageList(JSONArray historyMessage) {
JSONArray messageArJson;
try {
messageArJson = historyMessage.getJSONArray(0);
} catch (JSONException e) {
String str = safeConvertHistoryMessageToString(historyMessage);
throw new RuntimeException("could not extract element 0 (messageList) of retrieved historyMessage (historyMessage:\n" + str
+ "\n(end of historyMessage)\n)", e);
}
return messageArJson;
}
public boolean hasNext() {
synchronized (Iter.this.buffer) {
ensureNotInExceptionState();
if (Iter.this.buffer.isEmpty())
retrieveNextBatch();
return !Iter.this.buffer.isEmpty();
}
}
public Object next() {
synchronized (Iter.this.buffer) {
if (!hasNext()) {
throw new NoSuchElementException("there are no more elements in this iterator");
}
Object result = Iter.this.buffer.removeFirst();
return result;
}
}
private void ensureNotInExceptionState() {
if (caughtDuringRetrieval != null) {
throw new RuntimeException("an exception was caught already by a previous attempt to access this iterator", caughtDuringRetrieval);
}
}
public void remove() {
throw new UnsupportedOperationException(getClass().getName() + " doesn't support remove()");
}
}
}