Maatwebsite/Laravel-Excel View format - excel

I am loading Laravel view and exporting as Excel using Maatwebsite/Laravel-Excel but my data showing as text but i need to make it as decimal, number. how can i do this. I have already read the documentation but there i cant find solution.
$fileName = 'Receipt Register : From '.date('d-m-Y', strtotime($date_from)).' To '.date('d-m-Y', strtotime($date_to)).($itemDetails ? ' For Item '.$itemDetails->item_code : "");
Excel::create($fileName, function( $excel) use($date_from, $date_to, $request) {
$excel->sheet('Receipt-Register', function($sheet) use($date_from, $date_to, $request) {
$itemDetails = [];
$itemFilterData = [];
$result = Ledger::where('receive_quantity', '!=', NULL)
->where('receive_quantity', ">", 0)
->orderBy('date', 'ASC')
->orderBy('mrn_number',"ASC")
->whereNotNull('mrn_number')
->whereNotNull('mrn_id')
->orderBy('id', "ASC")
->with('department', 'item', 'itemGroup', 'mrn')
->has('mrn', ">", 0);
if($request->date_from) {
$date_from = date('Y-m-d', strtotime($request->date_from));
$result = $result->whereDate('date', '>=', $date_from);
}
if($request->date_to) {
$date_to = date('Y-m-d', strtotime($request->date_to));
$result = $result->whereDate('date', '<=', $date_to);
}
if($request->item_id){
$result = $result->where('item_id', '=', $request->item_id);
$itemDetails = Item::find($request->item_id);
}
$results = $result->get();
$sheet->loadView('export_view',[
'results' => $results,
'date_from' => $date_from,
'date_to' => $date_to,
'itemFilterData' => $itemFilterData
]);
});
})->download('xlsx');

Related

CakePhp 3 File Manipulation vulnerability with checkmarx

I keep getting a File Manipulation and File Disclosure report when i run my application through checkmarx. The report states that:
The input obtained via add in the file src/Controller/UploadsController.php at line 67 is used to determine the location of a file to be written into by add in the file src/Controller/UploadsController.php at line 67, potentially allowing an attacker to alter or corrupt the contents of that file, or create a new file altogether.
Code Snippet:
....
78. $filtered = filter_var_array($this->request->getData('bulk_name'), $args);
....
94. if(escapeshellcmd(escapeshellarg(move_uploaded_file($tmp_name, $destination)))) {
....
150. public function sanitizeData($input){
....
211. public function sanitize($string, $forceLowerCase = true, $anal = false) {
This is what I've tried doing but I keep getting the same issues in the report. What else can I do to bypass this? Ideas/Solutions are welcome.
public function add() {
$upload = $this->Uploads->newEntity();
if (escapeshellcmd(escapeshellarg($this->request->is('post')))) {
$args = array(
'tmp_name' => FILTER_SANITIZE_URL,
'error' => FILTER_VALIDATE_INT,
'name' => FILTER_SANITIZE_ENCODED,
'type' => FILTER_SANITIZE_SPECIAL_CHARS,
'size' => FILTER_SANITIZE_ENCODED,
);
$filtered = filter_var_array($this->request->getData('bulk_name'), $args);
if (!empty($filtered)) {
$file = $this->sanitizeData($filtered);
if (isset($file['flashMessage'])) {
$flashMessage = $file['flashMessage'];
$this->Flash->error($flashMessage, ['key' => 'error']);
} else {
$uploadDirectory = getcwd() . DS . 'files' . DS;
$fileName = $file['name'];
$upload = $this->Uploads->patchEntity($upload, $this->request->getData());
$upload->file_name = $fileName;
$tmp_name = $file['tmp_name'];
$destination = $uploadDirectory . $fileName;
if (escapeshellcmd(escapeshellarg(move_uploaded_file($tmp_name, $destination)))) {
$datasource = ConnectionManager::get("default");
$datasource->begin();
$saveUpload = $this->Uploads->save($upload);
if ($saveUpload) {
$session = $this->getRequest()->getSession();
$clientID = $session->read('Auth.User.client_id');
$userID = $session->read('Auth.User.id');
$lastSavedId = $saveUpload->id;
$baseName = basename($fileName);
$uploadedCSVFile = $uploadDirectory . $baseName;
$csvFile = fopen($uploadedCSVFile, "r");
$totalAmount = 0;
while (($row = fgetcsv($csvFile)) !== false) {
if ($row[1] < 1) continue;
$totalAmount += trim($row[1]);
}
fclose($csvFile);
$uploadEntry = $this->Uploads->UploadEntries->newEntity();
$entryData = array();
$entryData['upload_id'] = $lastSavedId;
$entryData['client_id'] = $clientID;
$entryData['user_id'] = $userID;
$entryData['amount'] = $totalAmount;
$entryData['status'] = 0;
$uploadEntry = $this->Uploads->UploadEntries->patchEntity($UploadEntry, $entryData);
$saveUploadEntries = $this->Uploads->UploadEntries->save($uploadEntry);
if ($saveUploadEntries) {
$datasource->commit();
$this->Flash->success('The upload has been saved.', ['key' => 'success']);
return $this->redirect('/');
}
$this->Flash->error(__('The upload could not be saved. Please, try again.'));
}
$this->Flash->error(__('The upload could not be saved. Please, try again.'));
}
$this->Flash->error(__('The upload could not be saved. Please, try again.'));
}
} else {
$this->Flash->error(__('Empty Upload', ['key' => 'error']));
}
}
$this->set(compact('upload'));
}
public function sanitizeData($input){
$args = array(
'tmp_name' => FILTER_SANITIZE_URL,
'error' => FILTER_VALIDATE_INT,
'name' => FILTER_SANITIZE_ENCODED,
'type' => FILTER_SANITIZE_SPECIAL_CHARS,
'size' => FILTER_SANITIZE_ENCODED,
);
$filtered = filter_var_array($input, $args);
$fileExtensionsAllowed = ['csv']; // These will be the only file extensions allowed
$mimes = array('application/vnd.ms-excel', 'text/plain', 'text/csv', 'text/tsv');
$fileName = $filtered['name'];
$fileSize = $filtered['size'];
$fileTmpName = $filtered['tmp_name'];
$fileType = $filtered['type'];
$fileError = $filtered['error'];
$file = explode('.', $fileName);//Split file name with extension
$ext = end($file); //get extension name
$fileExtension = strtolower($ext); //if change extension to lowercase
$output = array();
if (!is_readable($fileTmpName)) {
$output['flashMessage'] = 'File is not readable';
} elseif (!in_array($fileExtension, $fileExtensionsAllowed) && !in_array($fileType, $mimes)) {
$output['flashMessage'] = 'Unsupported File Type';
} elseif ($fileSize > 50000) {
$output['flashMessage'] = 'File is too large for upload';
} elseif (!$fileError == 0) {
$output['flashMessage'] = 'An error occurred';
}
$csvName = $file[0];
$newfilename = $csvName . date("YmdHis") . '.' . $ext; //new file name
$output['name'] = $this->sanitize($newfilename); //sanitize file name
$output['error'] = $fileError;
$output['tmp_name'] = $fileTmpName;
$output['type'] = $fileType;
$output['size'] = $fileSize;
return $output;
}
public function sanitize($string, $forceLowerCase = true, $anal = false) {
if($this->Auth->user()) {
$strip = array(".","~", "`", "!", "#", "#", "$", "%", "^", "&", "*", "(", ")", "_", "=", "+", "[", "{", "]", "}", "\\", "|", ";", ":", "\"", "'", "‘", "’", "“", "”", "–", "—", "—", "–", ",", "<", ">", "/", "?");
$clean = trim(str_replace($strip, "", strip_tags($string)));
$clean = preg_replace('/\s+/', "-", $clean);
$clean = ($anal) ? preg_replace("/[^a-zA-Z0-9]/", "", $clean) : $clean;
return ($forceLowerCase) ?
(function_exists('mb_strtolower')) ?
mb_strtolower($clean, 'UTF-8') :
strtolower($clean) :
$clean;
}
}

How to Programmatically place order in opencart

I am new to opencart. And now i am working on the order module.The concept is i have to place order externally. So as in the controller/checkout/confirm.php order placement i have placed the order. The order also successfully stored at the order table. But the problem is, the order is not shown at the admin page. I have searched lot for this issue, finally i found that the order is not placed properly.
My code is,
public function index() {
$redirect = '';
$this->load->model('account/address');
$address = $this->model_account_address->getAddress($this->customer->getAddressId());
if ((!$this->cart->hasProducts() && empty($this->session->data['vouchers'])) || (!$this->cart->hasStock() && !$this->config->get('config_stock_checkout'))) {
$redirect = $this->url->link('checkout/cart');
}
// Validate minimum quantity requirements.
$products = $this->cart->getProducts();
foreach ($products as $product) {
$product_total = 0;
foreach ($products as $product_2) {
if ($product_2['product_id'] == $product['product_id']) {
$product_total += $product_2['quantity'];
}
}
if ($product['minimum'] > $product_total) {
$redirect = $this->url->link('checkout/cart');
break;
}
}
if (!$redirect) {
$order_data = array();
$order_data['totals'] = array();
$total = 0;
$taxes = $this->cart->getTaxes();
$this->load->model('extension/extension');
$sort_order = array();
$results = $this->model_extension_extension->getExtensions('total');
foreach ($results as $key => $value) {
$sort_order[$key] = $this->config->get($value['code'] . '_sort_order');
}
array_multisort($sort_order, SORT_ASC, $results);
foreach ($results as $result) {
if ($this->config->get($result['code'] . '_status')) {
$this->load->model('total/' . $result['code']);
$this->{'model_total_' . $result['code']}->getTotal($order_data['totals'], $total, $taxes);
}
}
$sort_order = array();
foreach ($order_data['totals'] as $key => $value) {
$sort_order[$key] = $value['sort_order'];
}
array_multisort($sort_order, SORT_ASC, $order_data['totals']);
$this->load->language('checkout/checkout');
$order_data['invoice_prefix'] = $this->config->get('config_invoice_prefix');
$order_data['store_id'] = $this->config->get('config_store_id');
$order_data['store_name'] = $this->config->get('config_name');
if ($order_data['store_id']) {
$order_data['store_url'] = $this->config->get('config_url');
} else {
$order_data['store_url'] = HTTP_SERVER;
}
if ($this->customer->isLogged()) {
$this->load->model('account/customer');
$customer_info = $this->model_account_customer->getCustomer($this->customer->getId());
$order_data['customer_id'] = $this->customer->getId();
$order_data['customer_group_id'] = $customer_info['customer_group_id'];
$order_data['firstname'] = $customer_info['firstname'];
$order_data['lastname'] = $customer_info['lastname'];
$order_data['email'] = $customer_info['email'];
$order_data['telephone'] = $customer_info['telephone'];
$order_data['fax'] = $customer_info['fax'];
$order_data['custom_field'] = unserialize($customer_info['custom_field']);
} elseif (isset($this->session->data['guest'])) {
$order_data['customer_id'] = 0;
$order_data['customer_group_id'] = $this->session->data['guest']['customer_group_id'];
$order_data['firstname'] = $this->session->data['guest']['firstname'];
$order_data['lastname'] = $this->session->data['guest']['lastname'];
$order_data['email'] = $this->session->data['guest']['email'];
$order_data['telephone'] = $this->session->data['guest']['telephone'];
$order_data['fax'] = $this->session->data['guest']['fax'];
$order_data['custom_field'] = $this->session->data['guest']['custom_field'];
}
$order_data['payment_firstname'] = $address['firstname'];
$order_data['payment_lastname'] = $address['lastname'];
$order_data['payment_company'] = $address['company'];
$order_data['payment_address_1'] = $address['address_1'];
$order_data['payment_address_2'] = $address['address_2'];
$order_data['payment_city'] = $address['city'];
$order_data['payment_postcode'] = $address['postcode'];
$order_data['payment_zone'] = $address['zone'];
$order_data['payment_zone_id'] = $address['zone_id'];
$order_data['payment_country'] = $address['country'];
$order_data['payment_country_id'] = $address['country_id'];
$order_data['payment_address_format'] = $address['address_format'];
$order_data['payment_custom_field'] = $address['custom_field'];
if (isset($this->session->data['payment_method']['title'])) {
$order_data['payment_method'] = $this->session->data['payment_method']['title'];
} else {
$order_data['payment_method'] = '';
}
if (isset($this->session->data['payment_method']['code'])) {
$order_data['payment_code'] = $this->session->data['payment_method']['code'];
} else {
$order_data['payment_code'] = '';
}
if ($this->cart->hasShipping()) {
$order_data['shipping_firstname'] = $address['firstname'];
$order_data['shipping_lastname'] = $address['lastname'];
$order_data['shipping_company'] = $address['company'];
$order_data['shipping_address_1'] = $address['address_1'];
$order_data['shipping_address_2'] = $address['address_2'];
$order_data['shipping_city'] = $address['city'];
$order_data['shipping_postcode'] = $address['postcode'];
$order_data['shipping_zone'] = $address['zone'];
$order_data['shipping_zone_id'] = $address['zone_id'];
$order_data['shipping_country'] = $address['country'];
$order_data['shipping_country_id'] = $address['country_id'];
$order_data['shipping_address_format'] = $address['address_format'];
$order_data['shipping_custom_field'] = $address['custom_field'];
if (isset($this->session->data['shipping_method']['title'])) {
$order_data['shipping_method'] = $this->session->data['shipping_method']['title'];
} else {
$order_data['shipping_method'] = '';
}
if (isset($this->session->data['shipping_method']['code'])) {
$order_data['shipping_code'] = $this->session->data['shipping_method']['code'];
} else {
$order_data['shipping_code'] = '';
}
} else {
$order_data['shipping_firstname'] = '';
$order_data['shipping_lastname'] = '';
$order_data['shipping_company'] = '';
$order_data['shipping_address_1'] = '';
$order_data['shipping_address_2'] = '';
$order_data['shipping_city'] = '';
$order_data['shipping_postcode'] = '';
$order_data['shipping_zone'] = '';
$order_data['shipping_zone_id'] = '';
$order_data['shipping_country'] = '';
$order_data['shipping_country_id'] = '';
$order_data['shipping_address_format'] = '';
$order_data['shipping_custom_field'] = array();
$order_data['shipping_method'] = '';
$order_data['shipping_code'] = '';
}
$order_data['products'] = array();
foreach ($this->cart->getProducts() as $product) {
$option_data = array();
foreach ($product['option'] as $option) {
$option_data[] = array(
'product_option_id' => $option['product_option_id'],
'product_option_value_id' => $option['product_option_value_id'],
'option_id' => $option['option_id'],
'option_value_id' => $option['option_value_id'],
'name' => $option['name'],
'value' => $option['value'],
'type' => $option['type']
);
}
$order_data['products'][] = array(
'product_id' => $product['product_id'],
'name' => $product['name'],
'model' => $product['model'],
'option' => $option_data,
'download' => $product['download'],
'quantity' => $product['quantity'],
'subtract' => $product['subtract'],
'price' => $product['price'],
'total' => $product['total'],
'tax' => $this->tax->getTax($product['price'], $product['tax_class_id']),
'reward' => $product['reward']
);
}
// Gift Voucher
$order_data['vouchers'] = array();
if (!empty($this->session->data['vouchers'])) {
foreach ($this->session->data['vouchers'] as $voucher) {
$order_data['vouchers'][] = array(
'description' => $voucher['description'],
'code' => substr(md5(mt_rand()), 0, 10),
'to_name' => $voucher['to_name'],
'to_email' => $voucher['to_email'],
'from_name' => $voucher['from_name'],
'from_email' => $voucher['from_email'],
'voucher_theme_id' => $voucher['voucher_theme_id'],
'message' => $voucher['message'],
'amount' => $voucher['amount']
);
}
}
$order_data['comment'] = "";
$order_data['total'] = $total;
if (isset($this->request->cookie['tracking'])) {
$order_data['tracking'] = $this->request->cookie['tracking'];
$subtotal = $this->cart->getSubTotal();
// Affiliate
$this->load->model('affiliate/affiliate');
$affiliate_info = $this->model_affiliate_affiliate->getAffiliateByCode($this->request->cookie['tracking']);
if ($affiliate_info) {
$order_data['affiliate_id'] = $affiliate_info['affiliate_id'];
$order_data['commission'] = ($subtotal / 100) * $affiliate_info['commission'];
} else {
$order_data['affiliate_id'] = 0;
$order_data['commission'] = 0;
}
// Marketing
$this->load->model('checkout/marketing');
$marketing_info = $this->model_checkout_marketing->getMarketingByCode($this->request->cookie['tracking']);
if ($marketing_info) {
$order_data['marketing_id'] = $marketing_info['marketing_id'];
} else {
$order_data['marketing_id'] = 0;
}
} else {
$order_data['affiliate_id'] = 0;
$order_data['commission'] = 0;
$order_data['marketing_id'] = 0;
$order_data['tracking'] = '';
}
$order_data['language_id'] = $this->config->get('config_language_id');
$order_data['currency_id'] = $this->currency->getId();
$order_data['currency_code'] = $this->currency->getCode();
$order_data['currency_value'] = $this->currency->getValue($this->currency->getCode());
$order_data['ip'] = $this->request->server['REMOTE_ADDR'];
if (!empty($this->request->server['HTTP_X_FORWARDED_FOR'])) {
$order_data['forwarded_ip'] = $this->request->server['HTTP_X_FORWARDED_FOR'];
} elseif (!empty($this->request->server['HTTP_CLIENT_IP'])) {
$order_data['forwarded_ip'] = $this->request->server['HTTP_CLIENT_IP'];
} else {
$order_data['forwarded_ip'] = '';
}
if (isset($this->request->server['HTTP_USER_AGENT'])) {
$order_data['user_agent'] = $this->request->server['HTTP_USER_AGENT'];
} else {
$order_data['user_agent'] = '';
}
if (isset($this->request->server['HTTP_ACCEPT_LANGUAGE'])) {
$order_data['accept_language'] = $this->request->server['HTTP_ACCEPT_LANGUAGE'];
} else {
$order_data['accept_language'] = '';
}
$this->load->model('checkout/order');
$this->session->data['order_id'] = $this->model_checkout_order->addOrder($order_data);
$data['text_recurring_item'] = $this->language->get('text_recurring_item');
$data['text_payment_recurring'] = $this->language->get('text_payment_recurring');
$data['column_name'] = $this->language->get('column_name');
$data['column_model'] = $this->language->get('column_model');
$data['column_quantity'] = $this->language->get('column_quantity');
$data['column_price'] = $this->language->get('column_price');
$data['column_total'] = $this->language->get('column_total');
$this->load->model('tool/upload');
$data['products'] = array();
foreach ($this->cart->getProducts() as $product) {
$option_data = array();
foreach ($product['option'] as $option) {
if ($option['type'] != 'file') {
$value = $option['value'];
} else {
$upload_info = $this->model_tool_upload->getUploadByCode($option['value']);
if ($upload_info) {
$value = $upload_info['name'];
} else {
$value = '';
}
}
$option_data[] = array(
'name' => $option['name'],
'value' => (utf8_strlen($value) > 20 ? utf8_substr($value, 0, 20) . '..' : $value)
);
}
$recurring = '';
if ($product['recurring']) {
$frequencies = array(
'day' => $this->language->get('text_day'),
'week' => $this->language->get('text_week'),
'semi_month' => $this->language->get('text_semi_month'),
'month' => $this->language->get('text_month'),
'year' => $this->language->get('text_year'),
);
if ($product['recurring']['trial']) {
$recurring = sprintf($this->language->get('text_trial_description'), $this->currency->format($this->tax->calculate($product['recurring']['trial_price'] * $product['quantity'], $product['tax_class_id'], $this->config->get('config_tax'))), $product['recurring']['trial_cycle'], $frequencies[$product['recurring']['trial_frequency']], $product['recurring']['trial_duration']) . ' ';
}
if ($product['recurring']['duration']) {
$recurring .= sprintf($this->language->get('text_payment_description'), $this->currency->format($this->tax->calculate($product['recurring']['price'] * $product['quantity'], $product['tax_class_id'], $this->config->get('config_tax'))), $product['recurring']['cycle'], $frequencies[$product['recurring']['frequency']], $product['recurring']['duration']);
} else {
$recurring .= sprintf($this->language->get('text_payment_cancel'), $this->currency->format($this->tax->calculate($product['recurring']['price'] * $product['quantity'], $product['tax_class_id'], $this->config->get('config_tax'))), $product['recurring']['cycle'], $frequencies[$product['recurring']['frequency']], $product['recurring']['duration']);
}
}
$data['products'][] = array(
'key' => $product['key'],
'product_id' => $product['product_id'],
'name' => $product['name'],
'model' => $product['model'],
'option' => $option_data,
'recurring' => $recurring,
'quantity' => $product['quantity'],
'subtract' => $product['subtract'],
'price' => $this->currency->format($this->tax->calculate($product['price'], $product['tax_class_id'], $this->config->get('config_tax'))),
'total' => $this->currency->format($this->tax->calculate($product['price'], $product['tax_class_id'], $this->config->get('config_tax')) * $product['quantity']),
'href' => $this->url->link('product/product', 'product_id=' . $product['product_id']),
);
}
// Gift Voucher
$data['vouchers'] = array();
if (!empty($this->session->data['vouchers'])) {
foreach ($this->session->data['vouchers'] as $voucher) {
$data['vouchers'][] = array(
'description' => $voucher['description'],
'amount' => $this->currency->format($voucher['amount'])
);
}
}
$data['totals'] = array();
foreach ($order_data['totals'] as $total) {
$data['totals'][] = array(
'title' => $total['title'],
'text' => $this->currency->format($total['value']),
);
}
//$data['payment'] = $this->load->controller('payment/' . $this->session->data['payment_method']['code']);
} else {
$data['redirect'] = $redirect;
}
echo json_encode("success");
}
Is this correct format or still any process to do like updating order table or etc...
I really don't know what to do next.. Please someone guide me to get rid of this issue..
Thanks
Opencart admin display order which orders have order status is > 0. Did you check your order_status_id in database it will be 0.
That's the issue. How Opencart works, when you are at checkout - confirm page but you haven't confirm your order, Opencart already entered one entry for that order with order status id - 0.
After that when you confirm your order than your selected payment method - callback function (in mostly payment method(s)) update your order status using model > checkout > order function addOrderHistory().
So problem is that you added your order to Opencart but not updated it's order_status_id so after adding order add a function to your module with will update order status of last (or your added) order. For that you can check default payment methods.

phpcassa cassandra batch mutation

Can you provide an example of batch_mutate() function in phpcassa?
Cant understand how to work with this function and didnt found any enough information.
Also i want to know how to use it with counters
Thanks in advance.
"delete data from multiple keys":
function batch_remove($key=null, $columns=null, $super_column=null, $write_consistency_level=null) {
$timestamp = CassandraUtil::get_time();
$deletion = new cassandra_Deletion();
$deletion->timestamp = $timestamp;
if ($super_column !== null) $deletion->super_column = $this->pack_name($super_column, true);
else $deletion->super_column = null;
if ($columns !== null) {
$predicate = $this->create_slice_predicate($columns, '', '', false, self::DEFAULT_COLUMN_COUNT);
$deletion->predicate = $predicate;
}
$mutation = new cassandra_Mutation();
$mutation->deletion = $deletion;
if (is_array($key) && count($key) >= 1) {
$mut_map = array();
foreach($key as $v) {
$packed_key[$v] = $this->pack_key($v);
$mut_map[$v] = array($this->column_family => array($mutation));
}
return $this->pool->call("batch_mutate", $mut_map, $this->wcl($write_consistency_level));
} else return false;
}
//delete name1, name2, name3 from key1, key2 in a single call
$column_family->batch_remove(array(key1, key2), array(name1, name2, name3));
batch_mutate on counters is not available yet with PHPCassa: https://github.com/thobbs/phpcassa/issues/31
batch_mutate in action, as per the tutorial: http://thobbs.github.com/phpcassa/tutorial.html
"inserting data":
$row1 = array('name1' => 'val1', 'name2' => 'val2');
$row2 = array('foo' => 'bar');
$column_family->batch_insert(array('row1' => $row1, 'row2' => $row2);

How to retrieve all the reviews publicly available for an extension in the Google Chrome webstore - JSON & cross-domain issue

I'm interested in gathering/scraping data about the reviews earned by popular extensions available in the Chrome Webstore.
In particular, I need to retrieve the number of total reviews left for a specific extension and then retrieve all the reviews publicly available for this addon. My problem is the following: I cannot write a standard PHP Curl scraper since the data I'm interested in is available through json requests, in particular, I need to call:
https://chrome.google.com/reviews/components for the number of
reviews ('numRatings')
https://chrome.google.com/reviews/json/search
for the reviews ("comment")
I tried to write this:
<script src="http://code.jquery.com/jquery-latest.js"></script>
<script type="text/javascript">
function getReviews(extensionId, callback) {
var entities = [{'url' : 'http://chrome.google.com/extensions/permalink?id=' + extensionId}];
var param = {"searchSpecs":[{"requireComment":true,"entities": entities,"groups":["public_comment"],"matchExtraGroups":true,"sortBy":"quality","startIndex":10,"numResults":10,"includeNickNames":true}],"applicationId":94};
$.ajax({
type: 'POST',
url: 'https://chrome.google.com/reviews/json/search',
contentType: 'application/xml',
xhrFields: {withCredentials: true },
dataType: 'json',
data: 'req=' + JSON.stringify(param) + '&requestSource=widget'
}).success(callback);
}
</script>
<script type="text/javascript">
$(document).ready(getReviews('gighmmpiobklfepjocnamgkkbiglidom', function(reviews) { console.log(reviews); }));
</script>
I'm not very keen in jQuery/JSON(-P) and the code above is certainly wrong.
My questions are as follows:
How to bypass the same-domain policy? I tried YQL without success...
How to format my url/'data' to only retrieve the number of
reviews ('numRatings') on chrome.google.com/reviews/components and the reviews ('comments') on chrome.google.com/reviews/json/search for a specific extension identified by its id, e.g. gighmmpiobklfepjocnamgkkbiglidom?
I already accomplished this kind of scraping for popular Mozilla Addons using PHP and gathered the data I needed using a standard curl/XPath.
Thanks for your help!
1) The easiest way would be to create a Chrome extension;
2) See https://github.com/xpressyoo/MyExtensions
[...]
getComments : function() {
var entities = [];
//each(Ext.extensions, function(data, id) {
entities.push({'url' : 'http://chrome.google.com/extensions/permalink?id=' + this.hash});
//});
Ext.XHR['comments'] = new Ajax({
'method' : 'POST',
'encodeURI' : false, // Needed
'url' : 'https://chrome.google.com/reviews/json/search',
'headers' : {
'Content-type' : 'application/xml'
},
'parameters' : {
'req' : JSON.stringify({'searchSpecs' : [{'entities' : entities, 'groups' : ['public_comment'], 'matchExtraGroups' : true,"sortBy":"quality", 'startIndex' : 0, 'numResults' : 80, 'includeNickNames' : true}], 'applicationId' : 94 }) + '&requestSource=widget'
},
'onSuccess' : function(xhr) {
var json = xhr.responseJSON;
if(json && json.searchResults ) {
this.comments = {
'total' : Number(json.searchResults[0].numAnnotations.toString().replace(/,/, '').toInt()),
'latest' : json.searchResults[0].annotations ? json.searchResults[0].annotations[0] :{},
'previous' : this.comments.total || null,
'latestPrevious' : $merge(this.comments.latest) || null,
'new' : this.comments['new'] || false
}
Ext.XHR['comments'] = null;
}
}.bind(this)
}).send();
return this;
},
[...]
and
var nbreviews = this.comments.total; //The number of reviews
var latestcomment = (this.comments.latest0 && this.comments.latest0.comment ? this.comments.latest0.comment.replace(/\n/gi, '') : '');// get the latest comment
var nthcomment = (this.comments.latestn && this.comments.latestn.comment ? this.comments.latestn.comment.replace(/\n/gi, '') : '');//Get the nth comment
where:
'latestn' : json.searchResults[0].annotations ? json.searchResults[0].annotations[n] :{},
Here is a way of doing it in PHP with parallel cURL. This script scrapes all the extensions present in the Chrome webstore (ranked by popularity) and retrieves information such as:
Number of users
Number of star-ratings
Number of text reviews
Number of characters for each text-review (max 100 reviews scraped for each extension)
//GET URL
$url0 = "https://chrome.google.com/";
//AUTO LOOP
foreach(range(0, 705, 5) as $x) {
//Nb PAGES TO DOWNLOAD
$frompge = $x+1;
$topge = $x+5;
$nbpages = ($topge - $frompge)+1;
$zitems = $nbpages*20;
//MULTI cURL INIT
$mh = curl_multi_init();
$running = null;
//GENERATE URLs ARRAY
$urls = array();
for ($a = $frompge; $a <= $topge; $a++){
$aa = $url0 . 'webstore/list/most_popular/'. $a .'?category=ext';
$urls[] = $aa;
}
foreach ($urls as $name => $url)
{
$c[$name]=curl_init($url);
curl_setopt($c[$name], CURLOPT_HEADER, false);
curl_setopt($c[$name], CURLOPT_FAILONERROR, true);
curl_setopt($c[$name], CURLOPT_FOLLOWLOCATION, true);
curl_setopt($c[$name], CURLOPT_AUTOREFERER, true);
curl_setopt($c[$name], CURLOPT_RETURNTRANSFER, true);
curl_setopt($c[$name], CURLOPT_TIMEOUT, 10);
curl_multi_add_handle ($mh,$c[$name]);
}
// execute all queries simultaneously, and continue when all are complete
do {
curl_multi_exec($mh, $running);
} while ($running >0);
$html = array();
foreach ($urls as $name => $url)
{
$html[]=curl_multi_getcontent($c[$name]);
curl_multi_remove_handle($mh,$c[$name]);
curl_close($c[$name]);
}
curl_multi_close($mh);
for ($b = 0; $b <= $nbpages-1; $b++) {
// Parse the HTML information and return the results.
$dom = new DOMDocument();
#$dom->loadHtml($html[$b]);
$xpath = new DOMXPath($dom);
$links = $xpath->query("//a[contains(#class, 'title-a')]");
$result = array();
foreach ( $links as $item ) {
$newDom = new DOMDocument;
$newDom->appendChild($newDom->importNode($item,true));
$xpath = new DOMXPath( $newDom );
$cleaner = array(" users", " user", "(", ")", ","," ");
$data = str_replace($cleaner,"",trim($xpath->query("//script")->item(0)->nodeValue));
list($b1,$id,$b2,$b3,$b4,$name,$b5,$b6,$b7,$b8,$b9,$b10,$b11,$b12,$b13,$nbusers) = explode("\"", $data);
$label = str_replace(" ", "", strtolower(ereg_replace("[^A-Za-z0-9 ]", "", $name)));
//CATEGORIES (based on nb of users)
if($nbusers<100){$category = '1';$color = 'inherit';}
else if($nbusers>=100 && $nbusers<1000){$category = '2';$color = '#E6EEEE';}
else if($nbusers>=1000 && $nbusers<10000){$category = '3';$color = '#CDDEDE';}
else if($nbusers>=10000 && $nbusers<100000){$category = '4';$color = '#B5CDCD';}
else if($nbusers>=100000 && $nbusers<1000000){$category = '5';$color = '#9CBDBD';}
else if($nbusers == '1000000+'){$category = '6';$color = '#83ACAC';}
else{$category = '-9';}
/////////////////////////////////////////////LOOP REVIEWS
$extURL = 'http://chrome.google.com/extensions/permalink?id='.$id;
$c1 = curl_init('https://chrome.google.com/reviews/json/search');
$c1a = curl_init('https://chrome.google.com/reviews/json/search');
$c2 = curl_init('https://chrome.google.com/reviews/json/lookup');
$fields1 = http_build_query(array(
'req' => '{"searchSpecs":[{"requireComment":true,"entities":[{"url":"'.$extURL.'"}],"groups":["public_comment"],"matchExtraGroups":true,"sortBy":"quality","startIndex":0,"numResults":100,"includeNickNames":false}],"applicationId":94}',
));
$options1 = array(
CURLOPT_RETURNTRANSFER => true,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_POST => true,
CURLOPT_TIMEOUT => 10,
CURLOPT_POSTFIELDS => $fields1,
);
$fields1a = http_build_query(array(
'req' => '{"searchSpecs":[{"requireComment":true,"entities":[{"url":"'.$extURL.'"}],"groups":["public_comment"],"matchExtraGroups":true,"startIndex":0,"numResults":100,"includeNickNames":false}],"applicationId":94}',
));
$options1a = array(
CURLOPT_RETURNTRANSFER => true,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_POST => true,
CURLOPT_TIMEOUT => 10,
CURLOPT_POSTFIELDS => $fields1a,
);
$fields2 = http_build_query(array(
'req' => '{"entities":[{"url" : "'.$extURL.'", "includeAggregateInfo" : true}],"applicationId":94}',
));
$options2 = array(
CURLOPT_RETURNTRANSFER => true,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_POST => true,
CURLOPT_TIMEOUT => 10,
CURLOPT_POSTFIELDS => $fields2,
);
curl_setopt_array($c1, $options1);
curl_setopt_array($c1a, $options1a);
curl_setopt_array($c2, $options2);
$mh2 = curl_multi_init();
curl_multi_add_handle($mh2,$c1);
curl_multi_add_handle($mh2,$c1a);
curl_multi_add_handle($mh2,$c2);
$active = null;
do {
curl_multi_exec($mh2, $active);
} while ($active >0);
//close the handles$c1 = curl_init('https://chrome.google.com/reviews/json/search');
$json1=curl_multi_getcontent($c1);
$json1a=curl_multi_getcontent($c1a);
$json2=curl_multi_getcontent($c2);
curl_multi_remove_handle($mh2, $c1);
curl_multi_remove_handle($mh2, $c1a);
curl_multi_remove_handle($mh2, $c2);
curl_multi_close($mh2);
$data1 = json_decode(utf8_encode($json1), true);
$data1a = json_decode(utf8_encode($json1a), true);
$data2 = json_decode(utf8_encode($json2), true);
if ($data1['channelHeader']['errorCode']) return;
$nbreviews = $data1['searchResults'][0]['numAnnotations'];
if ($nbreviews > 100){$nbreviews2=100;}
else{$nbreviews2=$nbreviews;}
//Sum strings
$comments = $data1['searchResults'][0]['annotations'];
$sum =0;
foreach($comments as $comment){
$msg = preg_replace('/[\n\r\t]/', ' ', htmlspecialchars($comment['comment']));
$msg = str_replace(">", "", $msg);
$msg = str_replace(" ", "", $msg);
$strlen = strlen($msg);
$sum += $strlen;
}
$add = $sum;
$final = $add/$nbreviews2;
//Sum strings A
if ($data1a['channelHeader']['errorCode']) return;
$nbreviewsa = $data1a['searchResults'][0]['numAnnotations'];
$commentsa = $data1a['searchResults'][0]['annotations'];
$suma =0;
foreach($commentsa as $commenta){
$msga = preg_replace('/[\n\r\t]/', ' ', htmlspecialchars($commenta['comment']));
$msga = str_replace(">", "", $msga);
$msga = str_replace(" ", "", $msga);
$strlena = strlen($msga);
$suma += $strlena;
}
$adda = $suma;
$finala = $adda/$nbreviews2;
//Ratings
if ($data2['channelHeader']['errorCode']) return;
$nbratings = $data2['annotations'][0]['aggregateInfo']['numRatings'];
$nbstars = $data2['annotations'][0]['aggregateInfo']['averageRating'];
$delta = $nbratings - $nbreviews;
$ratio = $nbratings/$nbusers;
$ratio2 = $nbreviews/$nbusers;
////////////////////////////////////////////END LOOP REVIEWS
//PUT VALUES TOGETHER
$result[] = array($name,$label,$id,$category,$nbusers,$nbratings,$nbreviews,$nbreviewsa,$delta,$ratio,$ratio2,$nbstars,$nbreviews2,$add,$final,$adda,$finala);
}//END FOREACH
//print_r($result,false);
//DISPLAY RESULTS
for ($z = 0; $z <= 20; $z++) {
echo "<tr><td class=\"non\">" .$result[$z][0] . "</td><td class=\"non\">" .$result[$z][1] . "</td><td>" .$result[$z][3] . "</td><td>" .$result[$z][4] . "</td><td>" .$result[$z][5] . "</td><td>" .$result[$z][6] . "</td><td>" .$result[$z][7] . "</td><td>" .$result[$z][8] . "</td><td>" .$result[$z][9] . "</td><td>" .$result[$z][10] . "</td><td>" .$result[$z][11] . "</td><td>" .$result[$z][12] . "</td><td>" .$result[$z][13] . "</td><td>" .$result[$z][14] . "</td><td>" .$result[$z][15] . "</td><td>" .$result[$z][16] . "</td></tr>";
ob_flush();
flush();
}
}
}//END FOREACH

CakePHP Issue with Search and Pagination

I have a problem with my CakePhp Site.
My index is showing fine with pagination, but when I search the page, pagination is gone and the search results aren't showing (instead the index is showing without paging).
Any idea where my code is wrong?
class ItemsController extends AppController {
var $name = 'Items';
// load any helpers used in the views
var $helpers = array('Paginator', 'Html', 'Form', 'Javascript', 'Misc', 'Time', 'Tagcloud');
var $components = array('RequestHandler');
/**
* index()
* main index page for items
* url: /items/index
*/
function index() {
// get all options for form
$tags = $this->Item->Tag->find('list', array(
'fields'=>'id, name',
'order'=>'Tag.name',
'conditions'=> array(
'Tag.status'=>'1'
)
));
// add name to option
$tags = array(''=>'Tags') + $tags;
//pr($tags);
// if form submitted
if (!empty($this->data)) {
// if reset button pressed redirect to index page
if(isset($this->data['reset'])) {
$this->redirect(array('action'=>'index'));
}
// init
$url = '';
// remove search key if not set
if($this->data['search'] == '') {
unset($this->data['search']);
}
// loop through filters
foreach($this->data as $key=>$filter) {
// ignore submit button
if($key != 'filter') {
// init
$selected = '';
switch($key) {
case 'tag':
$selected = $tags[$filter];
break;
case 'search':
$selected = $filter;
break;
}
// if filter value is not empty
if(!empty($filter)) {
$selected = $this->slug($selected);
$url .= "/$key/$selected";
}
}
}
// redirect
$this->redirect('/items/index/'.$url);
} else {
// set form options
$this->data['tag'] = '';
$this->data['search'] = '';
}
// if any parameters have been passed
if(!empty($this->params['pass'])) {
// only select active items
$conditions = array('Item.status'=>1);
// get params
$params = $this->params['pass'];
// loop
foreach($params as $key=>$param) {
// get the filter value
if(isset($params[$key+1])) {
$value = $params[$key+1];
}
// switch on param
switch($param)
{
case 'tag':
// get tag
$tag = $this->Item->Tag->find('first', array(
'recursive' => 0,
'conditions' => array(
'Tag.slug'=>$value
)
));
// save value for form
$this->data['tag'] = $tag['Tag']['id'];
break;
case 'search':
// setup like clause
$conditions['Item.name LIKE'] = "%{$value}%";
// save search string for form
$this->data['search'] = str_replace('_', ' ', $value);
break;
}
}
//pr($conditions);
// get all items with param conditions
$items = $this->Item->find('all', array(
'order' => 'Item.name',
'conditions' => $conditions
));
// if tag filter has been set
if(isset($tag)) {
// loop through items
foreach($items as $key=>$item) {
// init
$found = FALSE;
// loop through tags
foreach($item['Tag'] as $k=>$g) {
// if the tag id matches the filter tag no need to continue
if($g['id'] == $tag['Tag']['id']) {
$found = TRUE;
break;
}
}
// if the tag was not found in items
if(!$found) {
// remove from list
unset($items[$key]);
}
}
}
} else {
// get all items from database where status = 1, order by name
$items = $this->Item->find('all', array(
'order' => 'Item.name',
'conditions' => array(
'Item.status'=>1
)
));
}
$this->paginate = array(
'limit' => 10
);
$data = $this->paginate('Item');
// set page title
$this->pageTitle = 'Index Page';
// set layout file
$this->layout = 'index';
// save the items in a variable for the view
$this->set(compact('data', 'tags', 'items'));
}
You'll want to pass your search conditions to the paginate functionality. You do this through the controller's paginate property.
function index() {
...
switch($param) {
...
case 'search':
$this->paginate['conditions']['Item.name LIKE'] = "%{$value}%";
More information on setting up pagination can be found here.

Resources