CakePhp 3 File Manipulation vulnerability with checkmarx - security

I keep getting a File Manipulation and File Disclosure report when i run my application through checkmarx. The report states that:
The input obtained via add in the file src/Controller/UploadsController.php at line 67 is used to determine the location of a file to be written into by add in the file src/Controller/UploadsController.php at line 67, potentially allowing an attacker to alter or corrupt the contents of that file, or create a new file altogether.
Code Snippet:
....
78. $filtered = filter_var_array($this->request->getData('bulk_name'), $args);
....
94. if(escapeshellcmd(escapeshellarg(move_uploaded_file($tmp_name, $destination)))) {
....
150. public function sanitizeData($input){
....
211. public function sanitize($string, $forceLowerCase = true, $anal = false) {
This is what I've tried doing but I keep getting the same issues in the report. What else can I do to bypass this? Ideas/Solutions are welcome.
public function add() {
$upload = $this->Uploads->newEntity();
if (escapeshellcmd(escapeshellarg($this->request->is('post')))) {
$args = array(
'tmp_name' => FILTER_SANITIZE_URL,
'error' => FILTER_VALIDATE_INT,
'name' => FILTER_SANITIZE_ENCODED,
'type' => FILTER_SANITIZE_SPECIAL_CHARS,
'size' => FILTER_SANITIZE_ENCODED,
);
$filtered = filter_var_array($this->request->getData('bulk_name'), $args);
if (!empty($filtered)) {
$file = $this->sanitizeData($filtered);
if (isset($file['flashMessage'])) {
$flashMessage = $file['flashMessage'];
$this->Flash->error($flashMessage, ['key' => 'error']);
} else {
$uploadDirectory = getcwd() . DS . 'files' . DS;
$fileName = $file['name'];
$upload = $this->Uploads->patchEntity($upload, $this->request->getData());
$upload->file_name = $fileName;
$tmp_name = $file['tmp_name'];
$destination = $uploadDirectory . $fileName;
if (escapeshellcmd(escapeshellarg(move_uploaded_file($tmp_name, $destination)))) {
$datasource = ConnectionManager::get("default");
$datasource->begin();
$saveUpload = $this->Uploads->save($upload);
if ($saveUpload) {
$session = $this->getRequest()->getSession();
$clientID = $session->read('Auth.User.client_id');
$userID = $session->read('Auth.User.id');
$lastSavedId = $saveUpload->id;
$baseName = basename($fileName);
$uploadedCSVFile = $uploadDirectory . $baseName;
$csvFile = fopen($uploadedCSVFile, "r");
$totalAmount = 0;
while (($row = fgetcsv($csvFile)) !== false) {
if ($row[1] < 1) continue;
$totalAmount += trim($row[1]);
}
fclose($csvFile);
$uploadEntry = $this->Uploads->UploadEntries->newEntity();
$entryData = array();
$entryData['upload_id'] = $lastSavedId;
$entryData['client_id'] = $clientID;
$entryData['user_id'] = $userID;
$entryData['amount'] = $totalAmount;
$entryData['status'] = 0;
$uploadEntry = $this->Uploads->UploadEntries->patchEntity($UploadEntry, $entryData);
$saveUploadEntries = $this->Uploads->UploadEntries->save($uploadEntry);
if ($saveUploadEntries) {
$datasource->commit();
$this->Flash->success('The upload has been saved.', ['key' => 'success']);
return $this->redirect('/');
}
$this->Flash->error(__('The upload could not be saved. Please, try again.'));
}
$this->Flash->error(__('The upload could not be saved. Please, try again.'));
}
$this->Flash->error(__('The upload could not be saved. Please, try again.'));
}
} else {
$this->Flash->error(__('Empty Upload', ['key' => 'error']));
}
}
$this->set(compact('upload'));
}
public function sanitizeData($input){
$args = array(
'tmp_name' => FILTER_SANITIZE_URL,
'error' => FILTER_VALIDATE_INT,
'name' => FILTER_SANITIZE_ENCODED,
'type' => FILTER_SANITIZE_SPECIAL_CHARS,
'size' => FILTER_SANITIZE_ENCODED,
);
$filtered = filter_var_array($input, $args);
$fileExtensionsAllowed = ['csv']; // These will be the only file extensions allowed
$mimes = array('application/vnd.ms-excel', 'text/plain', 'text/csv', 'text/tsv');
$fileName = $filtered['name'];
$fileSize = $filtered['size'];
$fileTmpName = $filtered['tmp_name'];
$fileType = $filtered['type'];
$fileError = $filtered['error'];
$file = explode('.', $fileName);//Split file name with extension
$ext = end($file); //get extension name
$fileExtension = strtolower($ext); //if change extension to lowercase
$output = array();
if (!is_readable($fileTmpName)) {
$output['flashMessage'] = 'File is not readable';
} elseif (!in_array($fileExtension, $fileExtensionsAllowed) && !in_array($fileType, $mimes)) {
$output['flashMessage'] = 'Unsupported File Type';
} elseif ($fileSize > 50000) {
$output['flashMessage'] = 'File is too large for upload';
} elseif (!$fileError == 0) {
$output['flashMessage'] = 'An error occurred';
}
$csvName = $file[0];
$newfilename = $csvName . date("YmdHis") . '.' . $ext; //new file name
$output['name'] = $this->sanitize($newfilename); //sanitize file name
$output['error'] = $fileError;
$output['tmp_name'] = $fileTmpName;
$output['type'] = $fileType;
$output['size'] = $fileSize;
return $output;
}
public function sanitize($string, $forceLowerCase = true, $anal = false) {
if($this->Auth->user()) {
$strip = array(".","~", "`", "!", "#", "#", "$", "%", "^", "&", "*", "(", ")", "_", "=", "+", "[", "{", "]", "}", "\\", "|", ";", ":", "\"", "'", "‘", "’", "“", "”", "–", "—", "—", "–", ",", "<", ">", "/", "?");
$clean = trim(str_replace($strip, "", strip_tags($string)));
$clean = preg_replace('/\s+/', "-", $clean);
$clean = ($anal) ? preg_replace("/[^a-zA-Z0-9]/", "", $clean) : $clean;
return ($forceLowerCase) ?
(function_exists('mb_strtolower')) ?
mb_strtolower($clean, 'UTF-8') :
strtolower($clean) :
$clean;
}
}

Related

Laravel pagination with raw query

When I put pagination(50); it shows WhereNull does not exist. and then I go make the whereNull into ctrl + /
and it show the next line which is OrderBy not exist. I seriously dont know where the problem is.. because I want to make pagination without using datatable from a DB::raw query.....
public function searchParticipation(Request $request){
ini_set('memory_limit','2G');
if ($request->method() != 'POST') {
return redirect()->route('ticketparticipation.view');
}
$replaceStartDate = $this->remove_string($request['start_date']);
$replaceEndDate = $this->remove_string($request['end_date']);
$valStartDate = strtotime($replaceStartDate) + (60*60*8);
$valEndDate = strtotime($replaceEndDate) + (60*60*24) + (60*60*8) - 1;
$event_id = $request['event_id'];
$category_id = $request['category_id'];
$ticket_number = $request['ticket_number'];
$full_name = $request['full_name'];
$dataEvent = array(
'event_id' => $event_id,
'category_id' => $category_id,
'ticket_number' => $ticket_number
);
$superadmins = User::UserOrganizer()->get();
$user_id = Session::get('user_id');
$roles = Session::get('roles');
if(empty(Session::get('roles'))){
auth()->logout();
return redirect('/admin/logout2');
}
$eventx = Event::query();
$eventx = $eventx->select('id', 'name');
if ($roles == 'Organizer-Admin') {
$eventx->Where('admin_id','=',$user_id);
$event = $eventx->Where('is_deleted','=','0')->get(); //->Where('is_active','=','1')
}elseif($roles == 'Organizer-Project'){
$eventx->Where('project_manager_id','=',$user_id);
$event = $eventx->Where('is_deleted','=','0')->get(); //->Where('is_active','=','1')
}elseif($roles == 'Organizer-Super-Admin'){
$eventx->Where('superadmin_id','=',$user_id);
$event = $eventx->Where('is_deleted','=','0')->get(); //->Where('is_active','=','1')
}elseif($roles == 'Superadmin-Organizer'){
$event = $eventx->Where('is_deleted','=','0')->get(); //->Where('is_active','=','1')
}
$data = array(
'user_id' => $user_id,
'roles' => $roles,
'date_start' => $request['start_date'],
'date_end' => $request['end_date']
);
if($data['roles'] == 'Organizer-Admin'){
$field = "event.admin_id";
}elseif($data['roles'] == 'Organizer-Project'){
$field = "event.project_manager_id";
}else{
$field = "event.superadmin_id";
}
$tCount = EventParticipation::Select('event_participation.id')
->join('event', 'event.id', '=', 'event_participation.event_id')
->join('categories', 'categories.id', 'event_participation.run_id');
if($event_id != ''){
$tCount = $tCount->Where("event_participation.event_id", "=", $event_id);
}
if($category_id != ''){
$tCount = $tCount->Where("event_participation.run_id", "=", $category_id);
}
if($ticket_number != ''){
$tCount = $tCount->Where(DB::raw('concat(event.ticket_number_prepend,"",event_participation.queue_id)') , '=' , $ticket_number);
}
if($full_name != ''){
$tCount = $tCount->Where("event_participation.full_name", 'LIKE' , '%'.$full_name.'%');
}
$tCount = $tCount->Where("event_participation.acceptance_date", "<>", 0)
->Where("event_participation.is_participating", "=", 1)
->Where("event_participation.is_deleted", "=", 0)
->OrderBy('event_participation.creation_date','ASC')
->get();
$eventDetail = Event::Select('name', 'registration_end', 'type','primary_currency')->Where('id', '=', $event_id)->first();
if($tCount->count() < 10000) {
$ticketParticipation = EventParticipation::Select(
DB::raw("if(epu.full_name is not null, epu.full_name, event_participation.full_name) as full_name"),
DB::raw("if(epu.address is not null, epu.address, event_participation.address) as address"),
DB::raw("if(epu.city is not null, epu.city, event_participation.city) as city"),
DB::raw("if(epu.postcode is not null, epu.postcode, event_participation.postcode) as postcode"),
DB::raw("if(epu.state is not null, epu.state, event_participation.state) as state"),
DB::raw("if(epu.country is not null, epu.country, event_participation.country) as country"),
DB::raw("if(epu.additional_info is not null, epu.additional_info, event_participation.additional_info) as additional_info"),
'event_participation.id', 'event_participation.delivery_company', 'event_participation.tracking_number', 'event_participation.run_id','event_participation.event_id', 'event_participation.local_transaction_number', 'event_participation.user_id',
'event_participation.nric_passport', 'event_participation.gender', 'event_participation.tshirt_size', 'event_participation.nationality',
'event_participation.email', 'event_participation.contact_number', 'event_participation.emergency_contact_name', 'event_participation.emergency_contact_number', 'event_participation.medical_conditions', 'event_participation.amount',
'event_participation.after_discount', 'event_participation.discount_id', 'event_participation.payment_type', 'event_participation.remarks',
'event.name AS eventname', 'event.ticket_number_prepend', 'categories.title AS run_title', 'event_participation.date_of_birth', 'event_participation.creation_date', 'event_participation.acceptance_date',
DB::raw("FROM_UNIXTIME(`event_participation`.`creation_date`,\"%d-%m-%Y %h:%i:%s %p\") AS `register_date`"),
DB::raw("CONCAT(event.`ticket_number_prepend`, '', event_participation.`queue_id`) AS `ticket_number`"),
'virtual_parcel_shipping_order.awb_id','virtual_parcel_shipping_order.awb_url','virtual_parcel_shipping_order.tracking_url','virtual_parcel_shipping_order.shipping_status',
'virtual_parcel_shipping_order.pick_up_date','virtual_parcel_shipping_order.parcel_content',
'virtual_parcel_shipping_order.status','virtual_parcel_shipping_order.response')//DB::raw("FROM_UNIXTIME(`event_participation`.`date_of_birth`,\"%d-%m-%Y\") AS `dob`"),
->join('event', 'event.id', '=', 'event_participation.event_id')
->join('categories', 'categories.id', 'event_participation.run_id')
->leftjoin('event_participation_report_status','participation_id','event_participation.id')
->leftjoin('virtual_parcel_shipping_order','participant_id','event_participation.id')
->leftjoin('event_participation_utf8 as epu', 'event_participation.id', 'epu.participation_id')->paginate(50); <<<<<<<< got problem when I add in paginate(50)
if($event_id != ''){
$ticketParticipation = $ticketParticipation->Where("event_participation.event_id", "=", $event_id);
}
if($category_id != ''){
$ticketParticipation = $ticketParticipation->Where("event_participation.run_id", "=", $category_id);
}
if($ticket_number != ''){
$ticketParticipation = $ticketParticipation->Where(DB::raw('concat(event.ticket_number_prepend,"",event_participation.queue_id)') , '=' , $ticket_number);
}
if($full_name != ''){
$ticketParticipation = $ticketParticipation->Where("event_participation.full_name", 'LIKE' , '%'.$full_name.'%');
}
$ticketParticipation = $ticketParticipation->Where("event_participation.acceptance_date", "<>", 0)
->Where("event_participation.is_participating", "=", 1)
->Where("event_participation.is_deleted", "=", 0)
->WhereNull("event_participation_report_status.participation_id")
->OrderBy('event_participation.queue_id', 'DESC')
->get()->chunk(1000);
$eventDiscount = EventDiscountCode::Select('id', 'event_id', 'amount', 'code')->Where('event_id', $event_id)->get();
return view('admin.organizer.participant_summary',['ticketParticipation'=>$ticketParticipation], compact('event', 'ticketParticipation', 'eventDiscount', 'dataEvent', 'eventDetail'));
}else{
return view('admin.organizer.participant_summaryv2', compact('event','data', 'dataEvent', 'eventDetail'));
}
}
I am seriously not sure where is the problem.....

Maatwebsite/Laravel-Excel View format

I am loading Laravel view and exporting as Excel using Maatwebsite/Laravel-Excel but my data showing as text but i need to make it as decimal, number. how can i do this. I have already read the documentation but there i cant find solution.
$fileName = 'Receipt Register : From '.date('d-m-Y', strtotime($date_from)).' To '.date('d-m-Y', strtotime($date_to)).($itemDetails ? ' For Item '.$itemDetails->item_code : "");
Excel::create($fileName, function( $excel) use($date_from, $date_to, $request) {
$excel->sheet('Receipt-Register', function($sheet) use($date_from, $date_to, $request) {
$itemDetails = [];
$itemFilterData = [];
$result = Ledger::where('receive_quantity', '!=', NULL)
->where('receive_quantity', ">", 0)
->orderBy('date', 'ASC')
->orderBy('mrn_number',"ASC")
->whereNotNull('mrn_number')
->whereNotNull('mrn_id')
->orderBy('id', "ASC")
->with('department', 'item', 'itemGroup', 'mrn')
->has('mrn', ">", 0);
if($request->date_from) {
$date_from = date('Y-m-d', strtotime($request->date_from));
$result = $result->whereDate('date', '>=', $date_from);
}
if($request->date_to) {
$date_to = date('Y-m-d', strtotime($request->date_to));
$result = $result->whereDate('date', '<=', $date_to);
}
if($request->item_id){
$result = $result->where('item_id', '=', $request->item_id);
$itemDetails = Item::find($request->item_id);
}
$results = $result->get();
$sheet->loadView('export_view',[
'results' => $results,
'date_from' => $date_from,
'date_to' => $date_to,
'itemFilterData' => $itemFilterData
]);
});
})->download('xlsx');

file_get_contents in wordpress function

This code works correctly and saves a remote image to localhost (this wordpress plugin save tmdb cast image to local address):
function dt_cast_2($id, $type, $limit = false)
{
$name = get_post_meta($id, "dt_cast", $single = true);
if ($type == "img") {
if ($limit) {
$val = explode("]", $name);
$passer = $newvalor = array();
foreach ($val as $valor) {
if (!empty($valor)) {
$passer[] = substr($valor, 1);
}
}
for ($h = 0; $h <= 500; $h++) {
$newval = explode(";", $passer[$h]);
$fotoor = $newval[0];
$actorpapel = explode(",", $newval[1]);
if (!empty($actorpapel[0])) {
if ($newval[0] == "null") {
$fotoor = DT_DIR_URI . '/assets/img/no_foto_cast.png';
} else {
$fotoor = 'https://image.tmdb.org/t/p/w90' . $newval[0];
$uploaddir = wp_upload_dir();
$uploadfile = $uploaddir['basedir'] . $newval[0];
if(!file_exists($uploadfile))
{
$contents= file_get_contents($fotoor);
$savefile = fopen($uploadfile, 'w');
fwrite($savefile, $contents);
fclose($savefile);
}
$fotoor = $uploaddir['baseurl'] . $newval[0];
}}}}}}
I have a problem in this function, the image is not saved to local.
Can someone give me the correct code?
function dt_image($name, $id, $size, $type = false, $return = false, $gtsml = false) {
$img = get_post_meta($id, $name, $single = true);
$val = explode("\n", $img);
$mgsl = array();
$count = 0;
foreach ($val as $valor) {
if (!empty($valor)) {
if (substr($valor, 0, 1) == "/") {
$mgsl[] = 'https://image.tmdb.org/t/p/' . $size . '' . $valor . '';
} else {
$mgsl[] = $valor;
}
$count++;
} else {
if ($name == "dt_poster" && $img == NULL) {
$mgsl[] = esc_url( DT_DIR_URI ) . '/assets/img/no_poster.png';
}
}
}
$fotoor = 'https://image.tmdb.org/t/p/w90' . $newval[0];
i replaced this code and work it.
$fotoor = 'https://image.tmdb.org/t/p/w90' . $newval[0];
$uploaddir = wp_upload_dir();
$uploadfile = $uploaddir['basedir'] . $newval[0];
if(!file_exists($uploadfile))
{
$contents= file_get_contents($fotoor);
$savefile = fopen($uploadfile, 'w');
fwrite($savefile, $contents);
fclose($savefile);
}
$fotoor = $uploaddir['baseurl'] . $newval[0];

How to Programmatically place order in opencart

I am new to opencart. And now i am working on the order module.The concept is i have to place order externally. So as in the controller/checkout/confirm.php order placement i have placed the order. The order also successfully stored at the order table. But the problem is, the order is not shown at the admin page. I have searched lot for this issue, finally i found that the order is not placed properly.
My code is,
public function index() {
$redirect = '';
$this->load->model('account/address');
$address = $this->model_account_address->getAddress($this->customer->getAddressId());
if ((!$this->cart->hasProducts() && empty($this->session->data['vouchers'])) || (!$this->cart->hasStock() && !$this->config->get('config_stock_checkout'))) {
$redirect = $this->url->link('checkout/cart');
}
// Validate minimum quantity requirements.
$products = $this->cart->getProducts();
foreach ($products as $product) {
$product_total = 0;
foreach ($products as $product_2) {
if ($product_2['product_id'] == $product['product_id']) {
$product_total += $product_2['quantity'];
}
}
if ($product['minimum'] > $product_total) {
$redirect = $this->url->link('checkout/cart');
break;
}
}
if (!$redirect) {
$order_data = array();
$order_data['totals'] = array();
$total = 0;
$taxes = $this->cart->getTaxes();
$this->load->model('extension/extension');
$sort_order = array();
$results = $this->model_extension_extension->getExtensions('total');
foreach ($results as $key => $value) {
$sort_order[$key] = $this->config->get($value['code'] . '_sort_order');
}
array_multisort($sort_order, SORT_ASC, $results);
foreach ($results as $result) {
if ($this->config->get($result['code'] . '_status')) {
$this->load->model('total/' . $result['code']);
$this->{'model_total_' . $result['code']}->getTotal($order_data['totals'], $total, $taxes);
}
}
$sort_order = array();
foreach ($order_data['totals'] as $key => $value) {
$sort_order[$key] = $value['sort_order'];
}
array_multisort($sort_order, SORT_ASC, $order_data['totals']);
$this->load->language('checkout/checkout');
$order_data['invoice_prefix'] = $this->config->get('config_invoice_prefix');
$order_data['store_id'] = $this->config->get('config_store_id');
$order_data['store_name'] = $this->config->get('config_name');
if ($order_data['store_id']) {
$order_data['store_url'] = $this->config->get('config_url');
} else {
$order_data['store_url'] = HTTP_SERVER;
}
if ($this->customer->isLogged()) {
$this->load->model('account/customer');
$customer_info = $this->model_account_customer->getCustomer($this->customer->getId());
$order_data['customer_id'] = $this->customer->getId();
$order_data['customer_group_id'] = $customer_info['customer_group_id'];
$order_data['firstname'] = $customer_info['firstname'];
$order_data['lastname'] = $customer_info['lastname'];
$order_data['email'] = $customer_info['email'];
$order_data['telephone'] = $customer_info['telephone'];
$order_data['fax'] = $customer_info['fax'];
$order_data['custom_field'] = unserialize($customer_info['custom_field']);
} elseif (isset($this->session->data['guest'])) {
$order_data['customer_id'] = 0;
$order_data['customer_group_id'] = $this->session->data['guest']['customer_group_id'];
$order_data['firstname'] = $this->session->data['guest']['firstname'];
$order_data['lastname'] = $this->session->data['guest']['lastname'];
$order_data['email'] = $this->session->data['guest']['email'];
$order_data['telephone'] = $this->session->data['guest']['telephone'];
$order_data['fax'] = $this->session->data['guest']['fax'];
$order_data['custom_field'] = $this->session->data['guest']['custom_field'];
}
$order_data['payment_firstname'] = $address['firstname'];
$order_data['payment_lastname'] = $address['lastname'];
$order_data['payment_company'] = $address['company'];
$order_data['payment_address_1'] = $address['address_1'];
$order_data['payment_address_2'] = $address['address_2'];
$order_data['payment_city'] = $address['city'];
$order_data['payment_postcode'] = $address['postcode'];
$order_data['payment_zone'] = $address['zone'];
$order_data['payment_zone_id'] = $address['zone_id'];
$order_data['payment_country'] = $address['country'];
$order_data['payment_country_id'] = $address['country_id'];
$order_data['payment_address_format'] = $address['address_format'];
$order_data['payment_custom_field'] = $address['custom_field'];
if (isset($this->session->data['payment_method']['title'])) {
$order_data['payment_method'] = $this->session->data['payment_method']['title'];
} else {
$order_data['payment_method'] = '';
}
if (isset($this->session->data['payment_method']['code'])) {
$order_data['payment_code'] = $this->session->data['payment_method']['code'];
} else {
$order_data['payment_code'] = '';
}
if ($this->cart->hasShipping()) {
$order_data['shipping_firstname'] = $address['firstname'];
$order_data['shipping_lastname'] = $address['lastname'];
$order_data['shipping_company'] = $address['company'];
$order_data['shipping_address_1'] = $address['address_1'];
$order_data['shipping_address_2'] = $address['address_2'];
$order_data['shipping_city'] = $address['city'];
$order_data['shipping_postcode'] = $address['postcode'];
$order_data['shipping_zone'] = $address['zone'];
$order_data['shipping_zone_id'] = $address['zone_id'];
$order_data['shipping_country'] = $address['country'];
$order_data['shipping_country_id'] = $address['country_id'];
$order_data['shipping_address_format'] = $address['address_format'];
$order_data['shipping_custom_field'] = $address['custom_field'];
if (isset($this->session->data['shipping_method']['title'])) {
$order_data['shipping_method'] = $this->session->data['shipping_method']['title'];
} else {
$order_data['shipping_method'] = '';
}
if (isset($this->session->data['shipping_method']['code'])) {
$order_data['shipping_code'] = $this->session->data['shipping_method']['code'];
} else {
$order_data['shipping_code'] = '';
}
} else {
$order_data['shipping_firstname'] = '';
$order_data['shipping_lastname'] = '';
$order_data['shipping_company'] = '';
$order_data['shipping_address_1'] = '';
$order_data['shipping_address_2'] = '';
$order_data['shipping_city'] = '';
$order_data['shipping_postcode'] = '';
$order_data['shipping_zone'] = '';
$order_data['shipping_zone_id'] = '';
$order_data['shipping_country'] = '';
$order_data['shipping_country_id'] = '';
$order_data['shipping_address_format'] = '';
$order_data['shipping_custom_field'] = array();
$order_data['shipping_method'] = '';
$order_data['shipping_code'] = '';
}
$order_data['products'] = array();
foreach ($this->cart->getProducts() as $product) {
$option_data = array();
foreach ($product['option'] as $option) {
$option_data[] = array(
'product_option_id' => $option['product_option_id'],
'product_option_value_id' => $option['product_option_value_id'],
'option_id' => $option['option_id'],
'option_value_id' => $option['option_value_id'],
'name' => $option['name'],
'value' => $option['value'],
'type' => $option['type']
);
}
$order_data['products'][] = array(
'product_id' => $product['product_id'],
'name' => $product['name'],
'model' => $product['model'],
'option' => $option_data,
'download' => $product['download'],
'quantity' => $product['quantity'],
'subtract' => $product['subtract'],
'price' => $product['price'],
'total' => $product['total'],
'tax' => $this->tax->getTax($product['price'], $product['tax_class_id']),
'reward' => $product['reward']
);
}
// Gift Voucher
$order_data['vouchers'] = array();
if (!empty($this->session->data['vouchers'])) {
foreach ($this->session->data['vouchers'] as $voucher) {
$order_data['vouchers'][] = array(
'description' => $voucher['description'],
'code' => substr(md5(mt_rand()), 0, 10),
'to_name' => $voucher['to_name'],
'to_email' => $voucher['to_email'],
'from_name' => $voucher['from_name'],
'from_email' => $voucher['from_email'],
'voucher_theme_id' => $voucher['voucher_theme_id'],
'message' => $voucher['message'],
'amount' => $voucher['amount']
);
}
}
$order_data['comment'] = "";
$order_data['total'] = $total;
if (isset($this->request->cookie['tracking'])) {
$order_data['tracking'] = $this->request->cookie['tracking'];
$subtotal = $this->cart->getSubTotal();
// Affiliate
$this->load->model('affiliate/affiliate');
$affiliate_info = $this->model_affiliate_affiliate->getAffiliateByCode($this->request->cookie['tracking']);
if ($affiliate_info) {
$order_data['affiliate_id'] = $affiliate_info['affiliate_id'];
$order_data['commission'] = ($subtotal / 100) * $affiliate_info['commission'];
} else {
$order_data['affiliate_id'] = 0;
$order_data['commission'] = 0;
}
// Marketing
$this->load->model('checkout/marketing');
$marketing_info = $this->model_checkout_marketing->getMarketingByCode($this->request->cookie['tracking']);
if ($marketing_info) {
$order_data['marketing_id'] = $marketing_info['marketing_id'];
} else {
$order_data['marketing_id'] = 0;
}
} else {
$order_data['affiliate_id'] = 0;
$order_data['commission'] = 0;
$order_data['marketing_id'] = 0;
$order_data['tracking'] = '';
}
$order_data['language_id'] = $this->config->get('config_language_id');
$order_data['currency_id'] = $this->currency->getId();
$order_data['currency_code'] = $this->currency->getCode();
$order_data['currency_value'] = $this->currency->getValue($this->currency->getCode());
$order_data['ip'] = $this->request->server['REMOTE_ADDR'];
if (!empty($this->request->server['HTTP_X_FORWARDED_FOR'])) {
$order_data['forwarded_ip'] = $this->request->server['HTTP_X_FORWARDED_FOR'];
} elseif (!empty($this->request->server['HTTP_CLIENT_IP'])) {
$order_data['forwarded_ip'] = $this->request->server['HTTP_CLIENT_IP'];
} else {
$order_data['forwarded_ip'] = '';
}
if (isset($this->request->server['HTTP_USER_AGENT'])) {
$order_data['user_agent'] = $this->request->server['HTTP_USER_AGENT'];
} else {
$order_data['user_agent'] = '';
}
if (isset($this->request->server['HTTP_ACCEPT_LANGUAGE'])) {
$order_data['accept_language'] = $this->request->server['HTTP_ACCEPT_LANGUAGE'];
} else {
$order_data['accept_language'] = '';
}
$this->load->model('checkout/order');
$this->session->data['order_id'] = $this->model_checkout_order->addOrder($order_data);
$data['text_recurring_item'] = $this->language->get('text_recurring_item');
$data['text_payment_recurring'] = $this->language->get('text_payment_recurring');
$data['column_name'] = $this->language->get('column_name');
$data['column_model'] = $this->language->get('column_model');
$data['column_quantity'] = $this->language->get('column_quantity');
$data['column_price'] = $this->language->get('column_price');
$data['column_total'] = $this->language->get('column_total');
$this->load->model('tool/upload');
$data['products'] = array();
foreach ($this->cart->getProducts() as $product) {
$option_data = array();
foreach ($product['option'] as $option) {
if ($option['type'] != 'file') {
$value = $option['value'];
} else {
$upload_info = $this->model_tool_upload->getUploadByCode($option['value']);
if ($upload_info) {
$value = $upload_info['name'];
} else {
$value = '';
}
}
$option_data[] = array(
'name' => $option['name'],
'value' => (utf8_strlen($value) > 20 ? utf8_substr($value, 0, 20) . '..' : $value)
);
}
$recurring = '';
if ($product['recurring']) {
$frequencies = array(
'day' => $this->language->get('text_day'),
'week' => $this->language->get('text_week'),
'semi_month' => $this->language->get('text_semi_month'),
'month' => $this->language->get('text_month'),
'year' => $this->language->get('text_year'),
);
if ($product['recurring']['trial']) {
$recurring = sprintf($this->language->get('text_trial_description'), $this->currency->format($this->tax->calculate($product['recurring']['trial_price'] * $product['quantity'], $product['tax_class_id'], $this->config->get('config_tax'))), $product['recurring']['trial_cycle'], $frequencies[$product['recurring']['trial_frequency']], $product['recurring']['trial_duration']) . ' ';
}
if ($product['recurring']['duration']) {
$recurring .= sprintf($this->language->get('text_payment_description'), $this->currency->format($this->tax->calculate($product['recurring']['price'] * $product['quantity'], $product['tax_class_id'], $this->config->get('config_tax'))), $product['recurring']['cycle'], $frequencies[$product['recurring']['frequency']], $product['recurring']['duration']);
} else {
$recurring .= sprintf($this->language->get('text_payment_cancel'), $this->currency->format($this->tax->calculate($product['recurring']['price'] * $product['quantity'], $product['tax_class_id'], $this->config->get('config_tax'))), $product['recurring']['cycle'], $frequencies[$product['recurring']['frequency']], $product['recurring']['duration']);
}
}
$data['products'][] = array(
'key' => $product['key'],
'product_id' => $product['product_id'],
'name' => $product['name'],
'model' => $product['model'],
'option' => $option_data,
'recurring' => $recurring,
'quantity' => $product['quantity'],
'subtract' => $product['subtract'],
'price' => $this->currency->format($this->tax->calculate($product['price'], $product['tax_class_id'], $this->config->get('config_tax'))),
'total' => $this->currency->format($this->tax->calculate($product['price'], $product['tax_class_id'], $this->config->get('config_tax')) * $product['quantity']),
'href' => $this->url->link('product/product', 'product_id=' . $product['product_id']),
);
}
// Gift Voucher
$data['vouchers'] = array();
if (!empty($this->session->data['vouchers'])) {
foreach ($this->session->data['vouchers'] as $voucher) {
$data['vouchers'][] = array(
'description' => $voucher['description'],
'amount' => $this->currency->format($voucher['amount'])
);
}
}
$data['totals'] = array();
foreach ($order_data['totals'] as $total) {
$data['totals'][] = array(
'title' => $total['title'],
'text' => $this->currency->format($total['value']),
);
}
//$data['payment'] = $this->load->controller('payment/' . $this->session->data['payment_method']['code']);
} else {
$data['redirect'] = $redirect;
}
echo json_encode("success");
}
Is this correct format or still any process to do like updating order table or etc...
I really don't know what to do next.. Please someone guide me to get rid of this issue..
Thanks
Opencart admin display order which orders have order status is > 0. Did you check your order_status_id in database it will be 0.
That's the issue. How Opencart works, when you are at checkout - confirm page but you haven't confirm your order, Opencart already entered one entry for that order with order status id - 0.
After that when you confirm your order than your selected payment method - callback function (in mostly payment method(s)) update your order status using model > checkout > order function addOrderHistory().
So problem is that you added your order to Opencart but not updated it's order_status_id so after adding order add a function to your module with will update order status of last (or your added) order. For that you can check default payment methods.

How to retrieve all the reviews publicly available for an extension in the Google Chrome webstore - JSON & cross-domain issue

I'm interested in gathering/scraping data about the reviews earned by popular extensions available in the Chrome Webstore.
In particular, I need to retrieve the number of total reviews left for a specific extension and then retrieve all the reviews publicly available for this addon. My problem is the following: I cannot write a standard PHP Curl scraper since the data I'm interested in is available through json requests, in particular, I need to call:
https://chrome.google.com/reviews/components for the number of
reviews ('numRatings')
https://chrome.google.com/reviews/json/search
for the reviews ("comment")
I tried to write this:
<script src="http://code.jquery.com/jquery-latest.js"></script>
<script type="text/javascript">
function getReviews(extensionId, callback) {
var entities = [{'url' : 'http://chrome.google.com/extensions/permalink?id=' + extensionId}];
var param = {"searchSpecs":[{"requireComment":true,"entities": entities,"groups":["public_comment"],"matchExtraGroups":true,"sortBy":"quality","startIndex":10,"numResults":10,"includeNickNames":true}],"applicationId":94};
$.ajax({
type: 'POST',
url: 'https://chrome.google.com/reviews/json/search',
contentType: 'application/xml',
xhrFields: {withCredentials: true },
dataType: 'json',
data: 'req=' + JSON.stringify(param) + '&requestSource=widget'
}).success(callback);
}
</script>
<script type="text/javascript">
$(document).ready(getReviews('gighmmpiobklfepjocnamgkkbiglidom', function(reviews) { console.log(reviews); }));
</script>
I'm not very keen in jQuery/JSON(-P) and the code above is certainly wrong.
My questions are as follows:
How to bypass the same-domain policy? I tried YQL without success...
How to format my url/'data' to only retrieve the number of
reviews ('numRatings') on chrome.google.com/reviews/components and the reviews ('comments') on chrome.google.com/reviews/json/search for a specific extension identified by its id, e.g. gighmmpiobklfepjocnamgkkbiglidom?
I already accomplished this kind of scraping for popular Mozilla Addons using PHP and gathered the data I needed using a standard curl/XPath.
Thanks for your help!
1) The easiest way would be to create a Chrome extension;
2) See https://github.com/xpressyoo/MyExtensions
[...]
getComments : function() {
var entities = [];
//each(Ext.extensions, function(data, id) {
entities.push({'url' : 'http://chrome.google.com/extensions/permalink?id=' + this.hash});
//});
Ext.XHR['comments'] = new Ajax({
'method' : 'POST',
'encodeURI' : false, // Needed
'url' : 'https://chrome.google.com/reviews/json/search',
'headers' : {
'Content-type' : 'application/xml'
},
'parameters' : {
'req' : JSON.stringify({'searchSpecs' : [{'entities' : entities, 'groups' : ['public_comment'], 'matchExtraGroups' : true,"sortBy":"quality", 'startIndex' : 0, 'numResults' : 80, 'includeNickNames' : true}], 'applicationId' : 94 }) + '&requestSource=widget'
},
'onSuccess' : function(xhr) {
var json = xhr.responseJSON;
if(json && json.searchResults ) {
this.comments = {
'total' : Number(json.searchResults[0].numAnnotations.toString().replace(/,/, '').toInt()),
'latest' : json.searchResults[0].annotations ? json.searchResults[0].annotations[0] :{},
'previous' : this.comments.total || null,
'latestPrevious' : $merge(this.comments.latest) || null,
'new' : this.comments['new'] || false
}
Ext.XHR['comments'] = null;
}
}.bind(this)
}).send();
return this;
},
[...]
and
var nbreviews = this.comments.total; //The number of reviews
var latestcomment = (this.comments.latest0 && this.comments.latest0.comment ? this.comments.latest0.comment.replace(/\n/gi, '') : '');// get the latest comment
var nthcomment = (this.comments.latestn && this.comments.latestn.comment ? this.comments.latestn.comment.replace(/\n/gi, '') : '');//Get the nth comment
where:
'latestn' : json.searchResults[0].annotations ? json.searchResults[0].annotations[n] :{},
Here is a way of doing it in PHP with parallel cURL. This script scrapes all the extensions present in the Chrome webstore (ranked by popularity) and retrieves information such as:
Number of users
Number of star-ratings
Number of text reviews
Number of characters for each text-review (max 100 reviews scraped for each extension)
//GET URL
$url0 = "https://chrome.google.com/";
//AUTO LOOP
foreach(range(0, 705, 5) as $x) {
//Nb PAGES TO DOWNLOAD
$frompge = $x+1;
$topge = $x+5;
$nbpages = ($topge - $frompge)+1;
$zitems = $nbpages*20;
//MULTI cURL INIT
$mh = curl_multi_init();
$running = null;
//GENERATE URLs ARRAY
$urls = array();
for ($a = $frompge; $a <= $topge; $a++){
$aa = $url0 . 'webstore/list/most_popular/'. $a .'?category=ext';
$urls[] = $aa;
}
foreach ($urls as $name => $url)
{
$c[$name]=curl_init($url);
curl_setopt($c[$name], CURLOPT_HEADER, false);
curl_setopt($c[$name], CURLOPT_FAILONERROR, true);
curl_setopt($c[$name], CURLOPT_FOLLOWLOCATION, true);
curl_setopt($c[$name], CURLOPT_AUTOREFERER, true);
curl_setopt($c[$name], CURLOPT_RETURNTRANSFER, true);
curl_setopt($c[$name], CURLOPT_TIMEOUT, 10);
curl_multi_add_handle ($mh,$c[$name]);
}
// execute all queries simultaneously, and continue when all are complete
do {
curl_multi_exec($mh, $running);
} while ($running >0);
$html = array();
foreach ($urls as $name => $url)
{
$html[]=curl_multi_getcontent($c[$name]);
curl_multi_remove_handle($mh,$c[$name]);
curl_close($c[$name]);
}
curl_multi_close($mh);
for ($b = 0; $b <= $nbpages-1; $b++) {
// Parse the HTML information and return the results.
$dom = new DOMDocument();
#$dom->loadHtml($html[$b]);
$xpath = new DOMXPath($dom);
$links = $xpath->query("//a[contains(#class, 'title-a')]");
$result = array();
foreach ( $links as $item ) {
$newDom = new DOMDocument;
$newDom->appendChild($newDom->importNode($item,true));
$xpath = new DOMXPath( $newDom );
$cleaner = array(" users", " user", "(", ")", ","," ");
$data = str_replace($cleaner,"",trim($xpath->query("//script")->item(0)->nodeValue));
list($b1,$id,$b2,$b3,$b4,$name,$b5,$b6,$b7,$b8,$b9,$b10,$b11,$b12,$b13,$nbusers) = explode("\"", $data);
$label = str_replace(" ", "", strtolower(ereg_replace("[^A-Za-z0-9 ]", "", $name)));
//CATEGORIES (based on nb of users)
if($nbusers<100){$category = '1';$color = 'inherit';}
else if($nbusers>=100 && $nbusers<1000){$category = '2';$color = '#E6EEEE';}
else if($nbusers>=1000 && $nbusers<10000){$category = '3';$color = '#CDDEDE';}
else if($nbusers>=10000 && $nbusers<100000){$category = '4';$color = '#B5CDCD';}
else if($nbusers>=100000 && $nbusers<1000000){$category = '5';$color = '#9CBDBD';}
else if($nbusers == '1000000+'){$category = '6';$color = '#83ACAC';}
else{$category = '-9';}
/////////////////////////////////////////////LOOP REVIEWS
$extURL = 'http://chrome.google.com/extensions/permalink?id='.$id;
$c1 = curl_init('https://chrome.google.com/reviews/json/search');
$c1a = curl_init('https://chrome.google.com/reviews/json/search');
$c2 = curl_init('https://chrome.google.com/reviews/json/lookup');
$fields1 = http_build_query(array(
'req' => '{"searchSpecs":[{"requireComment":true,"entities":[{"url":"'.$extURL.'"}],"groups":["public_comment"],"matchExtraGroups":true,"sortBy":"quality","startIndex":0,"numResults":100,"includeNickNames":false}],"applicationId":94}',
));
$options1 = array(
CURLOPT_RETURNTRANSFER => true,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_POST => true,
CURLOPT_TIMEOUT => 10,
CURLOPT_POSTFIELDS => $fields1,
);
$fields1a = http_build_query(array(
'req' => '{"searchSpecs":[{"requireComment":true,"entities":[{"url":"'.$extURL.'"}],"groups":["public_comment"],"matchExtraGroups":true,"startIndex":0,"numResults":100,"includeNickNames":false}],"applicationId":94}',
));
$options1a = array(
CURLOPT_RETURNTRANSFER => true,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_POST => true,
CURLOPT_TIMEOUT => 10,
CURLOPT_POSTFIELDS => $fields1a,
);
$fields2 = http_build_query(array(
'req' => '{"entities":[{"url" : "'.$extURL.'", "includeAggregateInfo" : true}],"applicationId":94}',
));
$options2 = array(
CURLOPT_RETURNTRANSFER => true,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_POST => true,
CURLOPT_TIMEOUT => 10,
CURLOPT_POSTFIELDS => $fields2,
);
curl_setopt_array($c1, $options1);
curl_setopt_array($c1a, $options1a);
curl_setopt_array($c2, $options2);
$mh2 = curl_multi_init();
curl_multi_add_handle($mh2,$c1);
curl_multi_add_handle($mh2,$c1a);
curl_multi_add_handle($mh2,$c2);
$active = null;
do {
curl_multi_exec($mh2, $active);
} while ($active >0);
//close the handles$c1 = curl_init('https://chrome.google.com/reviews/json/search');
$json1=curl_multi_getcontent($c1);
$json1a=curl_multi_getcontent($c1a);
$json2=curl_multi_getcontent($c2);
curl_multi_remove_handle($mh2, $c1);
curl_multi_remove_handle($mh2, $c1a);
curl_multi_remove_handle($mh2, $c2);
curl_multi_close($mh2);
$data1 = json_decode(utf8_encode($json1), true);
$data1a = json_decode(utf8_encode($json1a), true);
$data2 = json_decode(utf8_encode($json2), true);
if ($data1['channelHeader']['errorCode']) return;
$nbreviews = $data1['searchResults'][0]['numAnnotations'];
if ($nbreviews > 100){$nbreviews2=100;}
else{$nbreviews2=$nbreviews;}
//Sum strings
$comments = $data1['searchResults'][0]['annotations'];
$sum =0;
foreach($comments as $comment){
$msg = preg_replace('/[\n\r\t]/', ' ', htmlspecialchars($comment['comment']));
$msg = str_replace(">", "", $msg);
$msg = str_replace(" ", "", $msg);
$strlen = strlen($msg);
$sum += $strlen;
}
$add = $sum;
$final = $add/$nbreviews2;
//Sum strings A
if ($data1a['channelHeader']['errorCode']) return;
$nbreviewsa = $data1a['searchResults'][0]['numAnnotations'];
$commentsa = $data1a['searchResults'][0]['annotations'];
$suma =0;
foreach($commentsa as $commenta){
$msga = preg_replace('/[\n\r\t]/', ' ', htmlspecialchars($commenta['comment']));
$msga = str_replace(">", "", $msga);
$msga = str_replace(" ", "", $msga);
$strlena = strlen($msga);
$suma += $strlena;
}
$adda = $suma;
$finala = $adda/$nbreviews2;
//Ratings
if ($data2['channelHeader']['errorCode']) return;
$nbratings = $data2['annotations'][0]['aggregateInfo']['numRatings'];
$nbstars = $data2['annotations'][0]['aggregateInfo']['averageRating'];
$delta = $nbratings - $nbreviews;
$ratio = $nbratings/$nbusers;
$ratio2 = $nbreviews/$nbusers;
////////////////////////////////////////////END LOOP REVIEWS
//PUT VALUES TOGETHER
$result[] = array($name,$label,$id,$category,$nbusers,$nbratings,$nbreviews,$nbreviewsa,$delta,$ratio,$ratio2,$nbstars,$nbreviews2,$add,$final,$adda,$finala);
}//END FOREACH
//print_r($result,false);
//DISPLAY RESULTS
for ($z = 0; $z <= 20; $z++) {
echo "<tr><td class=\"non\">" .$result[$z][0] . "</td><td class=\"non\">" .$result[$z][1] . "</td><td>" .$result[$z][3] . "</td><td>" .$result[$z][4] . "</td><td>" .$result[$z][5] . "</td><td>" .$result[$z][6] . "</td><td>" .$result[$z][7] . "</td><td>" .$result[$z][8] . "</td><td>" .$result[$z][9] . "</td><td>" .$result[$z][10] . "</td><td>" .$result[$z][11] . "</td><td>" .$result[$z][12] . "</td><td>" .$result[$z][13] . "</td><td>" .$result[$z][14] . "</td><td>" .$result[$z][15] . "</td><td>" .$result[$z][16] . "</td></tr>";
ob_flush();
flush();
}
}
}//END FOREACH

Resources