Related
I keep getting a File Manipulation and File Disclosure report when i run my application through checkmarx. The report states that:
The input obtained via add in the file src/Controller/UploadsController.php at line 67 is used to determine the location of a file to be written into by add in the file src/Controller/UploadsController.php at line 67, potentially allowing an attacker to alter or corrupt the contents of that file, or create a new file altogether.
Code Snippet:
....
78. $filtered = filter_var_array($this->request->getData('bulk_name'), $args);
....
94. if(escapeshellcmd(escapeshellarg(move_uploaded_file($tmp_name, $destination)))) {
....
150. public function sanitizeData($input){
....
211. public function sanitize($string, $forceLowerCase = true, $anal = false) {
This is what I've tried doing but I keep getting the same issues in the report. What else can I do to bypass this? Ideas/Solutions are welcome.
public function add() {
$upload = $this->Uploads->newEntity();
if (escapeshellcmd(escapeshellarg($this->request->is('post')))) {
$args = array(
'tmp_name' => FILTER_SANITIZE_URL,
'error' => FILTER_VALIDATE_INT,
'name' => FILTER_SANITIZE_ENCODED,
'type' => FILTER_SANITIZE_SPECIAL_CHARS,
'size' => FILTER_SANITIZE_ENCODED,
);
$filtered = filter_var_array($this->request->getData('bulk_name'), $args);
if (!empty($filtered)) {
$file = $this->sanitizeData($filtered);
if (isset($file['flashMessage'])) {
$flashMessage = $file['flashMessage'];
$this->Flash->error($flashMessage, ['key' => 'error']);
} else {
$uploadDirectory = getcwd() . DS . 'files' . DS;
$fileName = $file['name'];
$upload = $this->Uploads->patchEntity($upload, $this->request->getData());
$upload->file_name = $fileName;
$tmp_name = $file['tmp_name'];
$destination = $uploadDirectory . $fileName;
if (escapeshellcmd(escapeshellarg(move_uploaded_file($tmp_name, $destination)))) {
$datasource = ConnectionManager::get("default");
$datasource->begin();
$saveUpload = $this->Uploads->save($upload);
if ($saveUpload) {
$session = $this->getRequest()->getSession();
$clientID = $session->read('Auth.User.client_id');
$userID = $session->read('Auth.User.id');
$lastSavedId = $saveUpload->id;
$baseName = basename($fileName);
$uploadedCSVFile = $uploadDirectory . $baseName;
$csvFile = fopen($uploadedCSVFile, "r");
$totalAmount = 0;
while (($row = fgetcsv($csvFile)) !== false) {
if ($row[1] < 1) continue;
$totalAmount += trim($row[1]);
}
fclose($csvFile);
$uploadEntry = $this->Uploads->UploadEntries->newEntity();
$entryData = array();
$entryData['upload_id'] = $lastSavedId;
$entryData['client_id'] = $clientID;
$entryData['user_id'] = $userID;
$entryData['amount'] = $totalAmount;
$entryData['status'] = 0;
$uploadEntry = $this->Uploads->UploadEntries->patchEntity($UploadEntry, $entryData);
$saveUploadEntries = $this->Uploads->UploadEntries->save($uploadEntry);
if ($saveUploadEntries) {
$datasource->commit();
$this->Flash->success('The upload has been saved.', ['key' => 'success']);
return $this->redirect('/');
}
$this->Flash->error(__('The upload could not be saved. Please, try again.'));
}
$this->Flash->error(__('The upload could not be saved. Please, try again.'));
}
$this->Flash->error(__('The upload could not be saved. Please, try again.'));
}
} else {
$this->Flash->error(__('Empty Upload', ['key' => 'error']));
}
}
$this->set(compact('upload'));
}
public function sanitizeData($input){
$args = array(
'tmp_name' => FILTER_SANITIZE_URL,
'error' => FILTER_VALIDATE_INT,
'name' => FILTER_SANITIZE_ENCODED,
'type' => FILTER_SANITIZE_SPECIAL_CHARS,
'size' => FILTER_SANITIZE_ENCODED,
);
$filtered = filter_var_array($input, $args);
$fileExtensionsAllowed = ['csv']; // These will be the only file extensions allowed
$mimes = array('application/vnd.ms-excel', 'text/plain', 'text/csv', 'text/tsv');
$fileName = $filtered['name'];
$fileSize = $filtered['size'];
$fileTmpName = $filtered['tmp_name'];
$fileType = $filtered['type'];
$fileError = $filtered['error'];
$file = explode('.', $fileName);//Split file name with extension
$ext = end($file); //get extension name
$fileExtension = strtolower($ext); //if change extension to lowercase
$output = array();
if (!is_readable($fileTmpName)) {
$output['flashMessage'] = 'File is not readable';
} elseif (!in_array($fileExtension, $fileExtensionsAllowed) && !in_array($fileType, $mimes)) {
$output['flashMessage'] = 'Unsupported File Type';
} elseif ($fileSize > 50000) {
$output['flashMessage'] = 'File is too large for upload';
} elseif (!$fileError == 0) {
$output['flashMessage'] = 'An error occurred';
}
$csvName = $file[0];
$newfilename = $csvName . date("YmdHis") . '.' . $ext; //new file name
$output['name'] = $this->sanitize($newfilename); //sanitize file name
$output['error'] = $fileError;
$output['tmp_name'] = $fileTmpName;
$output['type'] = $fileType;
$output['size'] = $fileSize;
return $output;
}
public function sanitize($string, $forceLowerCase = true, $anal = false) {
if($this->Auth->user()) {
$strip = array(".","~", "`", "!", "#", "#", "$", "%", "^", "&", "*", "(", ")", "_", "=", "+", "[", "{", "]", "}", "\\", "|", ";", ":", "\"", "'", "‘", "’", "“", "”", "–", "—", "—", "–", ",", "<", ">", "/", "?");
$clean = trim(str_replace($strip, "", strip_tags($string)));
$clean = preg_replace('/\s+/', "-", $clean);
$clean = ($anal) ? preg_replace("/[^a-zA-Z0-9]/", "", $clean) : $clean;
return ($forceLowerCase) ?
(function_exists('mb_strtolower')) ?
mb_strtolower($clean, 'UTF-8') :
strtolower($clean) :
$clean;
}
}
I am new to opencart. And now i am working on the order module.The concept is i have to place order externally. So as in the controller/checkout/confirm.php order placement i have placed the order. The order also successfully stored at the order table. But the problem is, the order is not shown at the admin page. I have searched lot for this issue, finally i found that the order is not placed properly.
My code is,
public function index() {
$redirect = '';
$this->load->model('account/address');
$address = $this->model_account_address->getAddress($this->customer->getAddressId());
if ((!$this->cart->hasProducts() && empty($this->session->data['vouchers'])) || (!$this->cart->hasStock() && !$this->config->get('config_stock_checkout'))) {
$redirect = $this->url->link('checkout/cart');
}
// Validate minimum quantity requirements.
$products = $this->cart->getProducts();
foreach ($products as $product) {
$product_total = 0;
foreach ($products as $product_2) {
if ($product_2['product_id'] == $product['product_id']) {
$product_total += $product_2['quantity'];
}
}
if ($product['minimum'] > $product_total) {
$redirect = $this->url->link('checkout/cart');
break;
}
}
if (!$redirect) {
$order_data = array();
$order_data['totals'] = array();
$total = 0;
$taxes = $this->cart->getTaxes();
$this->load->model('extension/extension');
$sort_order = array();
$results = $this->model_extension_extension->getExtensions('total');
foreach ($results as $key => $value) {
$sort_order[$key] = $this->config->get($value['code'] . '_sort_order');
}
array_multisort($sort_order, SORT_ASC, $results);
foreach ($results as $result) {
if ($this->config->get($result['code'] . '_status')) {
$this->load->model('total/' . $result['code']);
$this->{'model_total_' . $result['code']}->getTotal($order_data['totals'], $total, $taxes);
}
}
$sort_order = array();
foreach ($order_data['totals'] as $key => $value) {
$sort_order[$key] = $value['sort_order'];
}
array_multisort($sort_order, SORT_ASC, $order_data['totals']);
$this->load->language('checkout/checkout');
$order_data['invoice_prefix'] = $this->config->get('config_invoice_prefix');
$order_data['store_id'] = $this->config->get('config_store_id');
$order_data['store_name'] = $this->config->get('config_name');
if ($order_data['store_id']) {
$order_data['store_url'] = $this->config->get('config_url');
} else {
$order_data['store_url'] = HTTP_SERVER;
}
if ($this->customer->isLogged()) {
$this->load->model('account/customer');
$customer_info = $this->model_account_customer->getCustomer($this->customer->getId());
$order_data['customer_id'] = $this->customer->getId();
$order_data['customer_group_id'] = $customer_info['customer_group_id'];
$order_data['firstname'] = $customer_info['firstname'];
$order_data['lastname'] = $customer_info['lastname'];
$order_data['email'] = $customer_info['email'];
$order_data['telephone'] = $customer_info['telephone'];
$order_data['fax'] = $customer_info['fax'];
$order_data['custom_field'] = unserialize($customer_info['custom_field']);
} elseif (isset($this->session->data['guest'])) {
$order_data['customer_id'] = 0;
$order_data['customer_group_id'] = $this->session->data['guest']['customer_group_id'];
$order_data['firstname'] = $this->session->data['guest']['firstname'];
$order_data['lastname'] = $this->session->data['guest']['lastname'];
$order_data['email'] = $this->session->data['guest']['email'];
$order_data['telephone'] = $this->session->data['guest']['telephone'];
$order_data['fax'] = $this->session->data['guest']['fax'];
$order_data['custom_field'] = $this->session->data['guest']['custom_field'];
}
$order_data['payment_firstname'] = $address['firstname'];
$order_data['payment_lastname'] = $address['lastname'];
$order_data['payment_company'] = $address['company'];
$order_data['payment_address_1'] = $address['address_1'];
$order_data['payment_address_2'] = $address['address_2'];
$order_data['payment_city'] = $address['city'];
$order_data['payment_postcode'] = $address['postcode'];
$order_data['payment_zone'] = $address['zone'];
$order_data['payment_zone_id'] = $address['zone_id'];
$order_data['payment_country'] = $address['country'];
$order_data['payment_country_id'] = $address['country_id'];
$order_data['payment_address_format'] = $address['address_format'];
$order_data['payment_custom_field'] = $address['custom_field'];
if (isset($this->session->data['payment_method']['title'])) {
$order_data['payment_method'] = $this->session->data['payment_method']['title'];
} else {
$order_data['payment_method'] = '';
}
if (isset($this->session->data['payment_method']['code'])) {
$order_data['payment_code'] = $this->session->data['payment_method']['code'];
} else {
$order_data['payment_code'] = '';
}
if ($this->cart->hasShipping()) {
$order_data['shipping_firstname'] = $address['firstname'];
$order_data['shipping_lastname'] = $address['lastname'];
$order_data['shipping_company'] = $address['company'];
$order_data['shipping_address_1'] = $address['address_1'];
$order_data['shipping_address_2'] = $address['address_2'];
$order_data['shipping_city'] = $address['city'];
$order_data['shipping_postcode'] = $address['postcode'];
$order_data['shipping_zone'] = $address['zone'];
$order_data['shipping_zone_id'] = $address['zone_id'];
$order_data['shipping_country'] = $address['country'];
$order_data['shipping_country_id'] = $address['country_id'];
$order_data['shipping_address_format'] = $address['address_format'];
$order_data['shipping_custom_field'] = $address['custom_field'];
if (isset($this->session->data['shipping_method']['title'])) {
$order_data['shipping_method'] = $this->session->data['shipping_method']['title'];
} else {
$order_data['shipping_method'] = '';
}
if (isset($this->session->data['shipping_method']['code'])) {
$order_data['shipping_code'] = $this->session->data['shipping_method']['code'];
} else {
$order_data['shipping_code'] = '';
}
} else {
$order_data['shipping_firstname'] = '';
$order_data['shipping_lastname'] = '';
$order_data['shipping_company'] = '';
$order_data['shipping_address_1'] = '';
$order_data['shipping_address_2'] = '';
$order_data['shipping_city'] = '';
$order_data['shipping_postcode'] = '';
$order_data['shipping_zone'] = '';
$order_data['shipping_zone_id'] = '';
$order_data['shipping_country'] = '';
$order_data['shipping_country_id'] = '';
$order_data['shipping_address_format'] = '';
$order_data['shipping_custom_field'] = array();
$order_data['shipping_method'] = '';
$order_data['shipping_code'] = '';
}
$order_data['products'] = array();
foreach ($this->cart->getProducts() as $product) {
$option_data = array();
foreach ($product['option'] as $option) {
$option_data[] = array(
'product_option_id' => $option['product_option_id'],
'product_option_value_id' => $option['product_option_value_id'],
'option_id' => $option['option_id'],
'option_value_id' => $option['option_value_id'],
'name' => $option['name'],
'value' => $option['value'],
'type' => $option['type']
);
}
$order_data['products'][] = array(
'product_id' => $product['product_id'],
'name' => $product['name'],
'model' => $product['model'],
'option' => $option_data,
'download' => $product['download'],
'quantity' => $product['quantity'],
'subtract' => $product['subtract'],
'price' => $product['price'],
'total' => $product['total'],
'tax' => $this->tax->getTax($product['price'], $product['tax_class_id']),
'reward' => $product['reward']
);
}
// Gift Voucher
$order_data['vouchers'] = array();
if (!empty($this->session->data['vouchers'])) {
foreach ($this->session->data['vouchers'] as $voucher) {
$order_data['vouchers'][] = array(
'description' => $voucher['description'],
'code' => substr(md5(mt_rand()), 0, 10),
'to_name' => $voucher['to_name'],
'to_email' => $voucher['to_email'],
'from_name' => $voucher['from_name'],
'from_email' => $voucher['from_email'],
'voucher_theme_id' => $voucher['voucher_theme_id'],
'message' => $voucher['message'],
'amount' => $voucher['amount']
);
}
}
$order_data['comment'] = "";
$order_data['total'] = $total;
if (isset($this->request->cookie['tracking'])) {
$order_data['tracking'] = $this->request->cookie['tracking'];
$subtotal = $this->cart->getSubTotal();
// Affiliate
$this->load->model('affiliate/affiliate');
$affiliate_info = $this->model_affiliate_affiliate->getAffiliateByCode($this->request->cookie['tracking']);
if ($affiliate_info) {
$order_data['affiliate_id'] = $affiliate_info['affiliate_id'];
$order_data['commission'] = ($subtotal / 100) * $affiliate_info['commission'];
} else {
$order_data['affiliate_id'] = 0;
$order_data['commission'] = 0;
}
// Marketing
$this->load->model('checkout/marketing');
$marketing_info = $this->model_checkout_marketing->getMarketingByCode($this->request->cookie['tracking']);
if ($marketing_info) {
$order_data['marketing_id'] = $marketing_info['marketing_id'];
} else {
$order_data['marketing_id'] = 0;
}
} else {
$order_data['affiliate_id'] = 0;
$order_data['commission'] = 0;
$order_data['marketing_id'] = 0;
$order_data['tracking'] = '';
}
$order_data['language_id'] = $this->config->get('config_language_id');
$order_data['currency_id'] = $this->currency->getId();
$order_data['currency_code'] = $this->currency->getCode();
$order_data['currency_value'] = $this->currency->getValue($this->currency->getCode());
$order_data['ip'] = $this->request->server['REMOTE_ADDR'];
if (!empty($this->request->server['HTTP_X_FORWARDED_FOR'])) {
$order_data['forwarded_ip'] = $this->request->server['HTTP_X_FORWARDED_FOR'];
} elseif (!empty($this->request->server['HTTP_CLIENT_IP'])) {
$order_data['forwarded_ip'] = $this->request->server['HTTP_CLIENT_IP'];
} else {
$order_data['forwarded_ip'] = '';
}
if (isset($this->request->server['HTTP_USER_AGENT'])) {
$order_data['user_agent'] = $this->request->server['HTTP_USER_AGENT'];
} else {
$order_data['user_agent'] = '';
}
if (isset($this->request->server['HTTP_ACCEPT_LANGUAGE'])) {
$order_data['accept_language'] = $this->request->server['HTTP_ACCEPT_LANGUAGE'];
} else {
$order_data['accept_language'] = '';
}
$this->load->model('checkout/order');
$this->session->data['order_id'] = $this->model_checkout_order->addOrder($order_data);
$data['text_recurring_item'] = $this->language->get('text_recurring_item');
$data['text_payment_recurring'] = $this->language->get('text_payment_recurring');
$data['column_name'] = $this->language->get('column_name');
$data['column_model'] = $this->language->get('column_model');
$data['column_quantity'] = $this->language->get('column_quantity');
$data['column_price'] = $this->language->get('column_price');
$data['column_total'] = $this->language->get('column_total');
$this->load->model('tool/upload');
$data['products'] = array();
foreach ($this->cart->getProducts() as $product) {
$option_data = array();
foreach ($product['option'] as $option) {
if ($option['type'] != 'file') {
$value = $option['value'];
} else {
$upload_info = $this->model_tool_upload->getUploadByCode($option['value']);
if ($upload_info) {
$value = $upload_info['name'];
} else {
$value = '';
}
}
$option_data[] = array(
'name' => $option['name'],
'value' => (utf8_strlen($value) > 20 ? utf8_substr($value, 0, 20) . '..' : $value)
);
}
$recurring = '';
if ($product['recurring']) {
$frequencies = array(
'day' => $this->language->get('text_day'),
'week' => $this->language->get('text_week'),
'semi_month' => $this->language->get('text_semi_month'),
'month' => $this->language->get('text_month'),
'year' => $this->language->get('text_year'),
);
if ($product['recurring']['trial']) {
$recurring = sprintf($this->language->get('text_trial_description'), $this->currency->format($this->tax->calculate($product['recurring']['trial_price'] * $product['quantity'], $product['tax_class_id'], $this->config->get('config_tax'))), $product['recurring']['trial_cycle'], $frequencies[$product['recurring']['trial_frequency']], $product['recurring']['trial_duration']) . ' ';
}
if ($product['recurring']['duration']) {
$recurring .= sprintf($this->language->get('text_payment_description'), $this->currency->format($this->tax->calculate($product['recurring']['price'] * $product['quantity'], $product['tax_class_id'], $this->config->get('config_tax'))), $product['recurring']['cycle'], $frequencies[$product['recurring']['frequency']], $product['recurring']['duration']);
} else {
$recurring .= sprintf($this->language->get('text_payment_cancel'), $this->currency->format($this->tax->calculate($product['recurring']['price'] * $product['quantity'], $product['tax_class_id'], $this->config->get('config_tax'))), $product['recurring']['cycle'], $frequencies[$product['recurring']['frequency']], $product['recurring']['duration']);
}
}
$data['products'][] = array(
'key' => $product['key'],
'product_id' => $product['product_id'],
'name' => $product['name'],
'model' => $product['model'],
'option' => $option_data,
'recurring' => $recurring,
'quantity' => $product['quantity'],
'subtract' => $product['subtract'],
'price' => $this->currency->format($this->tax->calculate($product['price'], $product['tax_class_id'], $this->config->get('config_tax'))),
'total' => $this->currency->format($this->tax->calculate($product['price'], $product['tax_class_id'], $this->config->get('config_tax')) * $product['quantity']),
'href' => $this->url->link('product/product', 'product_id=' . $product['product_id']),
);
}
// Gift Voucher
$data['vouchers'] = array();
if (!empty($this->session->data['vouchers'])) {
foreach ($this->session->data['vouchers'] as $voucher) {
$data['vouchers'][] = array(
'description' => $voucher['description'],
'amount' => $this->currency->format($voucher['amount'])
);
}
}
$data['totals'] = array();
foreach ($order_data['totals'] as $total) {
$data['totals'][] = array(
'title' => $total['title'],
'text' => $this->currency->format($total['value']),
);
}
//$data['payment'] = $this->load->controller('payment/' . $this->session->data['payment_method']['code']);
} else {
$data['redirect'] = $redirect;
}
echo json_encode("success");
}
Is this correct format or still any process to do like updating order table or etc...
I really don't know what to do next.. Please someone guide me to get rid of this issue..
Thanks
Opencart admin display order which orders have order status is > 0. Did you check your order_status_id in database it will be 0.
That's the issue. How Opencart works, when you are at checkout - confirm page but you haven't confirm your order, Opencart already entered one entry for that order with order status id - 0.
After that when you confirm your order than your selected payment method - callback function (in mostly payment method(s)) update your order status using model > checkout > order function addOrderHistory().
So problem is that you added your order to Opencart but not updated it's order_status_id so after adding order add a function to your module with will update order status of last (or your added) order. For that you can check default payment methods.
Can you provide an example of batch_mutate() function in phpcassa?
Cant understand how to work with this function and didnt found any enough information.
Also i want to know how to use it with counters
Thanks in advance.
"delete data from multiple keys":
function batch_remove($key=null, $columns=null, $super_column=null, $write_consistency_level=null) {
$timestamp = CassandraUtil::get_time();
$deletion = new cassandra_Deletion();
$deletion->timestamp = $timestamp;
if ($super_column !== null) $deletion->super_column = $this->pack_name($super_column, true);
else $deletion->super_column = null;
if ($columns !== null) {
$predicate = $this->create_slice_predicate($columns, '', '', false, self::DEFAULT_COLUMN_COUNT);
$deletion->predicate = $predicate;
}
$mutation = new cassandra_Mutation();
$mutation->deletion = $deletion;
if (is_array($key) && count($key) >= 1) {
$mut_map = array();
foreach($key as $v) {
$packed_key[$v] = $this->pack_key($v);
$mut_map[$v] = array($this->column_family => array($mutation));
}
return $this->pool->call("batch_mutate", $mut_map, $this->wcl($write_consistency_level));
} else return false;
}
//delete name1, name2, name3 from key1, key2 in a single call
$column_family->batch_remove(array(key1, key2), array(name1, name2, name3));
batch_mutate on counters is not available yet with PHPCassa: https://github.com/thobbs/phpcassa/issues/31
batch_mutate in action, as per the tutorial: http://thobbs.github.com/phpcassa/tutorial.html
"inserting data":
$row1 = array('name1' => 'val1', 'name2' => 'val2');
$row2 = array('foo' => 'bar');
$column_family->batch_insert(array('row1' => $row1, 'row2' => $row2);
I'm interested in gathering/scraping data about the reviews earned by popular extensions available in the Chrome Webstore.
In particular, I need to retrieve the number of total reviews left for a specific extension and then retrieve all the reviews publicly available for this addon. My problem is the following: I cannot write a standard PHP Curl scraper since the data I'm interested in is available through json requests, in particular, I need to call:
https://chrome.google.com/reviews/components for the number of
reviews ('numRatings')
https://chrome.google.com/reviews/json/search
for the reviews ("comment")
I tried to write this:
<script src="http://code.jquery.com/jquery-latest.js"></script>
<script type="text/javascript">
function getReviews(extensionId, callback) {
var entities = [{'url' : 'http://chrome.google.com/extensions/permalink?id=' + extensionId}];
var param = {"searchSpecs":[{"requireComment":true,"entities": entities,"groups":["public_comment"],"matchExtraGroups":true,"sortBy":"quality","startIndex":10,"numResults":10,"includeNickNames":true}],"applicationId":94};
$.ajax({
type: 'POST',
url: 'https://chrome.google.com/reviews/json/search',
contentType: 'application/xml',
xhrFields: {withCredentials: true },
dataType: 'json',
data: 'req=' + JSON.stringify(param) + '&requestSource=widget'
}).success(callback);
}
</script>
<script type="text/javascript">
$(document).ready(getReviews('gighmmpiobklfepjocnamgkkbiglidom', function(reviews) { console.log(reviews); }));
</script>
I'm not very keen in jQuery/JSON(-P) and the code above is certainly wrong.
My questions are as follows:
How to bypass the same-domain policy? I tried YQL without success...
How to format my url/'data' to only retrieve the number of
reviews ('numRatings') on chrome.google.com/reviews/components and the reviews ('comments') on chrome.google.com/reviews/json/search for a specific extension identified by its id, e.g. gighmmpiobklfepjocnamgkkbiglidom?
I already accomplished this kind of scraping for popular Mozilla Addons using PHP and gathered the data I needed using a standard curl/XPath.
Thanks for your help!
1) The easiest way would be to create a Chrome extension;
2) See https://github.com/xpressyoo/MyExtensions
[...]
getComments : function() {
var entities = [];
//each(Ext.extensions, function(data, id) {
entities.push({'url' : 'http://chrome.google.com/extensions/permalink?id=' + this.hash});
//});
Ext.XHR['comments'] = new Ajax({
'method' : 'POST',
'encodeURI' : false, // Needed
'url' : 'https://chrome.google.com/reviews/json/search',
'headers' : {
'Content-type' : 'application/xml'
},
'parameters' : {
'req' : JSON.stringify({'searchSpecs' : [{'entities' : entities, 'groups' : ['public_comment'], 'matchExtraGroups' : true,"sortBy":"quality", 'startIndex' : 0, 'numResults' : 80, 'includeNickNames' : true}], 'applicationId' : 94 }) + '&requestSource=widget'
},
'onSuccess' : function(xhr) {
var json = xhr.responseJSON;
if(json && json.searchResults ) {
this.comments = {
'total' : Number(json.searchResults[0].numAnnotations.toString().replace(/,/, '').toInt()),
'latest' : json.searchResults[0].annotations ? json.searchResults[0].annotations[0] :{},
'previous' : this.comments.total || null,
'latestPrevious' : $merge(this.comments.latest) || null,
'new' : this.comments['new'] || false
}
Ext.XHR['comments'] = null;
}
}.bind(this)
}).send();
return this;
},
[...]
and
var nbreviews = this.comments.total; //The number of reviews
var latestcomment = (this.comments.latest0 && this.comments.latest0.comment ? this.comments.latest0.comment.replace(/\n/gi, '') : '');// get the latest comment
var nthcomment = (this.comments.latestn && this.comments.latestn.comment ? this.comments.latestn.comment.replace(/\n/gi, '') : '');//Get the nth comment
where:
'latestn' : json.searchResults[0].annotations ? json.searchResults[0].annotations[n] :{},
Here is a way of doing it in PHP with parallel cURL. This script scrapes all the extensions present in the Chrome webstore (ranked by popularity) and retrieves information such as:
Number of users
Number of star-ratings
Number of text reviews
Number of characters for each text-review (max 100 reviews scraped for each extension)
//GET URL
$url0 = "https://chrome.google.com/";
//AUTO LOOP
foreach(range(0, 705, 5) as $x) {
//Nb PAGES TO DOWNLOAD
$frompge = $x+1;
$topge = $x+5;
$nbpages = ($topge - $frompge)+1;
$zitems = $nbpages*20;
//MULTI cURL INIT
$mh = curl_multi_init();
$running = null;
//GENERATE URLs ARRAY
$urls = array();
for ($a = $frompge; $a <= $topge; $a++){
$aa = $url0 . 'webstore/list/most_popular/'. $a .'?category=ext';
$urls[] = $aa;
}
foreach ($urls as $name => $url)
{
$c[$name]=curl_init($url);
curl_setopt($c[$name], CURLOPT_HEADER, false);
curl_setopt($c[$name], CURLOPT_FAILONERROR, true);
curl_setopt($c[$name], CURLOPT_FOLLOWLOCATION, true);
curl_setopt($c[$name], CURLOPT_AUTOREFERER, true);
curl_setopt($c[$name], CURLOPT_RETURNTRANSFER, true);
curl_setopt($c[$name], CURLOPT_TIMEOUT, 10);
curl_multi_add_handle ($mh,$c[$name]);
}
// execute all queries simultaneously, and continue when all are complete
do {
curl_multi_exec($mh, $running);
} while ($running >0);
$html = array();
foreach ($urls as $name => $url)
{
$html[]=curl_multi_getcontent($c[$name]);
curl_multi_remove_handle($mh,$c[$name]);
curl_close($c[$name]);
}
curl_multi_close($mh);
for ($b = 0; $b <= $nbpages-1; $b++) {
// Parse the HTML information and return the results.
$dom = new DOMDocument();
#$dom->loadHtml($html[$b]);
$xpath = new DOMXPath($dom);
$links = $xpath->query("//a[contains(#class, 'title-a')]");
$result = array();
foreach ( $links as $item ) {
$newDom = new DOMDocument;
$newDom->appendChild($newDom->importNode($item,true));
$xpath = new DOMXPath( $newDom );
$cleaner = array(" users", " user", "(", ")", ","," ");
$data = str_replace($cleaner,"",trim($xpath->query("//script")->item(0)->nodeValue));
list($b1,$id,$b2,$b3,$b4,$name,$b5,$b6,$b7,$b8,$b9,$b10,$b11,$b12,$b13,$nbusers) = explode("\"", $data);
$label = str_replace(" ", "", strtolower(ereg_replace("[^A-Za-z0-9 ]", "", $name)));
//CATEGORIES (based on nb of users)
if($nbusers<100){$category = '1';$color = 'inherit';}
else if($nbusers>=100 && $nbusers<1000){$category = '2';$color = '#E6EEEE';}
else if($nbusers>=1000 && $nbusers<10000){$category = '3';$color = '#CDDEDE';}
else if($nbusers>=10000 && $nbusers<100000){$category = '4';$color = '#B5CDCD';}
else if($nbusers>=100000 && $nbusers<1000000){$category = '5';$color = '#9CBDBD';}
else if($nbusers == '1000000+'){$category = '6';$color = '#83ACAC';}
else{$category = '-9';}
/////////////////////////////////////////////LOOP REVIEWS
$extURL = 'http://chrome.google.com/extensions/permalink?id='.$id;
$c1 = curl_init('https://chrome.google.com/reviews/json/search');
$c1a = curl_init('https://chrome.google.com/reviews/json/search');
$c2 = curl_init('https://chrome.google.com/reviews/json/lookup');
$fields1 = http_build_query(array(
'req' => '{"searchSpecs":[{"requireComment":true,"entities":[{"url":"'.$extURL.'"}],"groups":["public_comment"],"matchExtraGroups":true,"sortBy":"quality","startIndex":0,"numResults":100,"includeNickNames":false}],"applicationId":94}',
));
$options1 = array(
CURLOPT_RETURNTRANSFER => true,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_POST => true,
CURLOPT_TIMEOUT => 10,
CURLOPT_POSTFIELDS => $fields1,
);
$fields1a = http_build_query(array(
'req' => '{"searchSpecs":[{"requireComment":true,"entities":[{"url":"'.$extURL.'"}],"groups":["public_comment"],"matchExtraGroups":true,"startIndex":0,"numResults":100,"includeNickNames":false}],"applicationId":94}',
));
$options1a = array(
CURLOPT_RETURNTRANSFER => true,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_POST => true,
CURLOPT_TIMEOUT => 10,
CURLOPT_POSTFIELDS => $fields1a,
);
$fields2 = http_build_query(array(
'req' => '{"entities":[{"url" : "'.$extURL.'", "includeAggregateInfo" : true}],"applicationId":94}',
));
$options2 = array(
CURLOPT_RETURNTRANSFER => true,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_POST => true,
CURLOPT_TIMEOUT => 10,
CURLOPT_POSTFIELDS => $fields2,
);
curl_setopt_array($c1, $options1);
curl_setopt_array($c1a, $options1a);
curl_setopt_array($c2, $options2);
$mh2 = curl_multi_init();
curl_multi_add_handle($mh2,$c1);
curl_multi_add_handle($mh2,$c1a);
curl_multi_add_handle($mh2,$c2);
$active = null;
do {
curl_multi_exec($mh2, $active);
} while ($active >0);
//close the handles$c1 = curl_init('https://chrome.google.com/reviews/json/search');
$json1=curl_multi_getcontent($c1);
$json1a=curl_multi_getcontent($c1a);
$json2=curl_multi_getcontent($c2);
curl_multi_remove_handle($mh2, $c1);
curl_multi_remove_handle($mh2, $c1a);
curl_multi_remove_handle($mh2, $c2);
curl_multi_close($mh2);
$data1 = json_decode(utf8_encode($json1), true);
$data1a = json_decode(utf8_encode($json1a), true);
$data2 = json_decode(utf8_encode($json2), true);
if ($data1['channelHeader']['errorCode']) return;
$nbreviews = $data1['searchResults'][0]['numAnnotations'];
if ($nbreviews > 100){$nbreviews2=100;}
else{$nbreviews2=$nbreviews;}
//Sum strings
$comments = $data1['searchResults'][0]['annotations'];
$sum =0;
foreach($comments as $comment){
$msg = preg_replace('/[\n\r\t]/', ' ', htmlspecialchars($comment['comment']));
$msg = str_replace(">", "", $msg);
$msg = str_replace(" ", "", $msg);
$strlen = strlen($msg);
$sum += $strlen;
}
$add = $sum;
$final = $add/$nbreviews2;
//Sum strings A
if ($data1a['channelHeader']['errorCode']) return;
$nbreviewsa = $data1a['searchResults'][0]['numAnnotations'];
$commentsa = $data1a['searchResults'][0]['annotations'];
$suma =0;
foreach($commentsa as $commenta){
$msga = preg_replace('/[\n\r\t]/', ' ', htmlspecialchars($commenta['comment']));
$msga = str_replace(">", "", $msga);
$msga = str_replace(" ", "", $msga);
$strlena = strlen($msga);
$suma += $strlena;
}
$adda = $suma;
$finala = $adda/$nbreviews2;
//Ratings
if ($data2['channelHeader']['errorCode']) return;
$nbratings = $data2['annotations'][0]['aggregateInfo']['numRatings'];
$nbstars = $data2['annotations'][0]['aggregateInfo']['averageRating'];
$delta = $nbratings - $nbreviews;
$ratio = $nbratings/$nbusers;
$ratio2 = $nbreviews/$nbusers;
////////////////////////////////////////////END LOOP REVIEWS
//PUT VALUES TOGETHER
$result[] = array($name,$label,$id,$category,$nbusers,$nbratings,$nbreviews,$nbreviewsa,$delta,$ratio,$ratio2,$nbstars,$nbreviews2,$add,$final,$adda,$finala);
}//END FOREACH
//print_r($result,false);
//DISPLAY RESULTS
for ($z = 0; $z <= 20; $z++) {
echo "<tr><td class=\"non\">" .$result[$z][0] . "</td><td class=\"non\">" .$result[$z][1] . "</td><td>" .$result[$z][3] . "</td><td>" .$result[$z][4] . "</td><td>" .$result[$z][5] . "</td><td>" .$result[$z][6] . "</td><td>" .$result[$z][7] . "</td><td>" .$result[$z][8] . "</td><td>" .$result[$z][9] . "</td><td>" .$result[$z][10] . "</td><td>" .$result[$z][11] . "</td><td>" .$result[$z][12] . "</td><td>" .$result[$z][13] . "</td><td>" .$result[$z][14] . "</td><td>" .$result[$z][15] . "</td><td>" .$result[$z][16] . "</td></tr>";
ob_flush();
flush();
}
}
}//END FOREACH
I have a problem with my CakePhp Site.
My index is showing fine with pagination, but when I search the page, pagination is gone and the search results aren't showing (instead the index is showing without paging).
Any idea where my code is wrong?
class ItemsController extends AppController {
var $name = 'Items';
// load any helpers used in the views
var $helpers = array('Paginator', 'Html', 'Form', 'Javascript', 'Misc', 'Time', 'Tagcloud');
var $components = array('RequestHandler');
/**
* index()
* main index page for items
* url: /items/index
*/
function index() {
// get all options for form
$tags = $this->Item->Tag->find('list', array(
'fields'=>'id, name',
'order'=>'Tag.name',
'conditions'=> array(
'Tag.status'=>'1'
)
));
// add name to option
$tags = array(''=>'Tags') + $tags;
//pr($tags);
// if form submitted
if (!empty($this->data)) {
// if reset button pressed redirect to index page
if(isset($this->data['reset'])) {
$this->redirect(array('action'=>'index'));
}
// init
$url = '';
// remove search key if not set
if($this->data['search'] == '') {
unset($this->data['search']);
}
// loop through filters
foreach($this->data as $key=>$filter) {
// ignore submit button
if($key != 'filter') {
// init
$selected = '';
switch($key) {
case 'tag':
$selected = $tags[$filter];
break;
case 'search':
$selected = $filter;
break;
}
// if filter value is not empty
if(!empty($filter)) {
$selected = $this->slug($selected);
$url .= "/$key/$selected";
}
}
}
// redirect
$this->redirect('/items/index/'.$url);
} else {
// set form options
$this->data['tag'] = '';
$this->data['search'] = '';
}
// if any parameters have been passed
if(!empty($this->params['pass'])) {
// only select active items
$conditions = array('Item.status'=>1);
// get params
$params = $this->params['pass'];
// loop
foreach($params as $key=>$param) {
// get the filter value
if(isset($params[$key+1])) {
$value = $params[$key+1];
}
// switch on param
switch($param)
{
case 'tag':
// get tag
$tag = $this->Item->Tag->find('first', array(
'recursive' => 0,
'conditions' => array(
'Tag.slug'=>$value
)
));
// save value for form
$this->data['tag'] = $tag['Tag']['id'];
break;
case 'search':
// setup like clause
$conditions['Item.name LIKE'] = "%{$value}%";
// save search string for form
$this->data['search'] = str_replace('_', ' ', $value);
break;
}
}
//pr($conditions);
// get all items with param conditions
$items = $this->Item->find('all', array(
'order' => 'Item.name',
'conditions' => $conditions
));
// if tag filter has been set
if(isset($tag)) {
// loop through items
foreach($items as $key=>$item) {
// init
$found = FALSE;
// loop through tags
foreach($item['Tag'] as $k=>$g) {
// if the tag id matches the filter tag no need to continue
if($g['id'] == $tag['Tag']['id']) {
$found = TRUE;
break;
}
}
// if the tag was not found in items
if(!$found) {
// remove from list
unset($items[$key]);
}
}
}
} else {
// get all items from database where status = 1, order by name
$items = $this->Item->find('all', array(
'order' => 'Item.name',
'conditions' => array(
'Item.status'=>1
)
));
}
$this->paginate = array(
'limit' => 10
);
$data = $this->paginate('Item');
// set page title
$this->pageTitle = 'Index Page';
// set layout file
$this->layout = 'index';
// save the items in a variable for the view
$this->set(compact('data', 'tags', 'items'));
}
You'll want to pass your search conditions to the paginate functionality. You do this through the controller's paginate property.
function index() {
...
switch($param) {
...
case 'search':
$this->paginate['conditions']['Item.name LIKE'] = "%{$value}%";
More information on setting up pagination can be found here.