I'm trying to get data from a website that does not have an API documentation. I used selenium to login and navigate. Today I found the data I need, it appears after a post request.
There's any way I can get the exact post parameters to do it in my node program?
Data I need
edit 1:
the code that works from console browser scrap the data from the page and save a csv file:
const timeElapsed = Date.now();
const today = new Date(timeElapsed);
let updateTime;
let dataStudents = "";
let anoSerie;
let turma;
let prova;
let disciplina;
class AlunoProva {
constructor(nome, Status, anoSerie, turma, prova, disciplina, modalidadeEnsino, data) {
this.nome = nome;
this.Status = Status;
this.anoSerie = anoSerie;
this.turma = turma;
this.prova = prova;
this.disciplina = disciplina;
this.modalidadeEnsino = modalidadeEnsino;
this.data = data;
}
}
var saveData = (function () {
var a = document.createElement("a");
document.body.appendChild(a);
a.style = "display: none";
return function (data, fileName) {
blob = new Blob([data], { type: "ext/csv;charset=utf-8;" }),
url = window.URL.createObjectURL(blob);
a.href = url;
a.download = fileName;
a.click();
window.URL.revokeObjectURL(url);
};
}());
const atualState = "Finalizado - Digital, Não iniciado, Em progresso - Digital"
function getTurma() {
let Turma = prompt("Informe a letra da turma atual", "A");
while (Turma == null || Turma == "") {
alert("Truma inválida! Por favor, informe uma turma válida.");
Turma = prompt("Informe a letra da turma atual", "A");
}
return Turma;
}
turma = getTurma().toUpperCase();
function getCombo(id) {
let combo = document.getElementById(id);
let dataComboSelected = combo.options[combo.selectedIndex].text;
return dataComboSelected;
}
String.prototype.hashCode = function () {
var hash = 0, i, chr;
if (this.length === 0) return hash;
for (i = 0; i < this.length; i++) {
chr = this.charCodeAt(i);
hash = ((hash << 5) - hash) + chr;
hash |= 0; // Convert to 32bit integer
}
return hash;
};
let aluno = new AlunoProva()
function dataMount(data) {
if (data !== "ESTUDANTE" && data !== "PARTICIPAÇÃO")
if (!atualState.includes(data)) {
aluno.nome = data;
dataStudents += data + ",";
}
else {
aluno.Status = data;
dataStudents += data + ",";
aluno.anoSerie = anoSerie;
dataStudents += anoSerie + ","
aluno.turma = turma;
dataStudents += turma + ","
aluno.prova = prova;
dataStudents += prova + ","
aluno.disciplina = disciplina;
dataStudents += disciplina + ","
if (anoSerie.includes("EF")) {
aluno.modalidadeEnsino = "Fundamental";
dataStudents += "Fundamental,"
}
else {
aluno.modalidadeEnsino = "Médio";
dataStudents += "Médio,"
}
aluno.data = today.toLocaleDateString();
dataStudents += today.toLocaleDateString() + "\n"
}
}
prova = getCombo('selectnn74f9d641c4DADOS.VL_FILTRO_AVALIACAO')
disciplina = getCombo('select3lw7fc885326DADOS.VL_FILTRO_DISCIPLINA')
anoSerie = getCombo('selecty0fuc3bed07cDADOS.VL_FILTRO_ETAPA')
Array.from($$("body table tr *")).forEach((el) => {
el = String(el.innerHTML)
if (el.indexOf('Atualizado') === 0) {
updateTime = el;
} else if (el.length != 0 && el != undefined) {
dataMount(el);
}
});
let hashname = (anoSerie + turma + prova + disciplina + JSON.stringify(today.toLocaleDateString())).toString();
fileName = `${anoSerie} - ${turma} - ${prova} - ${disciplina}"${hashname.hashCode()}".csv`
saveData(dataStudents, fileName);
I'm having a problem with selectors on node, just rebember that I'm not conected with the site, I'm simulating a navigation with selenium. All the frameworks I checked to help me with selectors needed that I provide an url, but I can't, the only way I found to enter the system is from selenium navigation to simulating a real user.
Related
Hello i'm have some problem with BIP32
i use BIP32 library in nodejs and dart(flutter)
in dart(flutter) BIP32 derivePath show different privatekey but not of all mnemonic is just incorrect some mnemonic
in 1000 mnemonic different privatekey 1-2 mnemonic
below is dart derivePath,,
what is inccorect in this code:
Help me pls.
BIP32 derivePath(String path) {
final regex = new RegExp(r"^(m\/)?(\d+'?\/)*\d+'?$");
if (!regex.hasMatch(path)) throw new ArgumentError("Expected BIP32 Path");
List<String> splitPath = path.split("/");
if (splitPath[0] == "m") {
if (parentFingerprint != 0) throw new ArgumentError("Expected master, got child");
splitPath = splitPath.sublist(1);
print("splitPath: "+ splitPath);
}
print("splitPath: "+ splitPath);
return splitPath.fold(this, (BIP32 prevHd,String indexStr) {
int index;
if (indexStr.substring(indexStr.length - 1) == "'") {
index = int.parse(indexStr.substring(0, indexStr.length - 1));
return prevHd.deriveHardened(index);
} else {
index = int.parse(indexStr);
return prevHd.derive(index);
}
});
}
BIP32 derive(int index) {
if (index > UINT32_MAX || index < 0) throw new ArgumentError("Expected UInt32");
final isHardened = index >= HIGHEST_BIT;
Uint8List data = new Uint8List(37);
if (isHardened) {
if (isNeutered()) {
throw new ArgumentError("Missing private key for hardened child key");
}
data[0] = 0x00;
data.setRange(1, 33, privateKey);
data.buffer.asByteData().setUint32(33, index);
} else {
data.setRange(0, 33, publicKey);
data.buffer.asByteData().setUint32(33, index);
}
final I = hmacSHA512(chainCode, data);
final IL = I.sublist(0, 32);
final IR = I.sublist(32);
if (!ecc.isPrivate(IL)) {
return derive(index + 1);
}
BIP32 hd;
if (!isNeutered()) {
final ki = ecc.privateAdd(privateKey, IL);
if (ki == null) return derive(index + 1);
hd = BIP32.fromPrivateKey(ki, IR, network);
} else {
final ki = ecc.pointAddScalar(publicKey, IL, true);
if (ki == null) return derive(index + 1);
hd = BIP32.fromPublicKey(ki, IR, network);
}
hd.depth = depth + 1;
hd.index = index;
hd.parentFingerprint = fingerprint.buffer.asByteData().getUint32(0);
return hd;
}
BIP32 deriveHardened(int index) {
if (index > UINT31_MAX || index < 0) throw new ArgumentError("Expected UInt31");
return this.derive(index + HIGHEST_BIT);
}
OK can solved this problem problem occur in the ecuve function bytes is less than 32 bytes just padding 0 in the left equal 32 bytes just solved it !
I've tried implementing thishttps://www.paragon-inc.com/resources/blogs-posts/easy_excel_interaction_pt6 on an ASP.NET MVC 5 Application.
//SEE CODE BELOW
[HttpPost]
public ActionResult Upload(HttpPostedFileBase file)
{
var regPIN = DB.AspNetUsers.Where(i => i.Id == user.Id).Select(i => i.registrationPIN).FirstOrDefault();
if (file != null && file.ContentLength > 0)
{
var extension = Path.GetExtension(file.FileName);
var excelFile = Path.Combine(Server.MapPath("~/App_Data/BulkImports"),regPIN + extension);
if (System.IO.File.Exists(excelFile))
{
System.IO.File.Delete(excelFile);
}
else if (file.ContentType == "application/vnd.ms-excel" || file.ContentType == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
{
file.SaveAs(excelFile);//WORKS FINE
//BEGINING OF IMPORT
FileInfo eFile = new FileInfo(excelFile);
using (var excelPackage = new ExcelPackage(eFile))
{
if (!eFile.Name.EndsWith("xlsx"))//Return ModelState.AddModelError()
{ ModelState.AddModelError("", "Incompartible Excel Document. Please use MSExcel 2007 and Above!"); }
else
{
var worksheet = excelPackage.Workbook.Worksheets[1];
if (worksheet == null) { ModelState.AddModelError("", "Wrong Excel Format!"); }// return ImportResults.WrongFormat;
else
{
var lastRow = worksheet.Dimension.End.Row;
while (lastRow >= 1)
{
var range = worksheet.Cells[lastRow, 1, lastRow, 3];
if (range.Any(c => c.Value != null))
{ break; }
lastRow--;
}
using (var db = new BlackBox_FinaleEntities())// var db = new BlackBox_FinaleEntities())
{
for (var row = 2; row <= lastRow; row++)
{
var newPerson = new personalDetails
{
identificationType = worksheet.Cells[row, 1].Value.ToString(),
idNumber = worksheet.Cells[row, 2].Value.ToString(),
idSerial = worksheet.Cells[row, 3].Value.ToString(),
fullName = worksheet.Cells[row, 4].Value.ToString(),
dob = DateTime.Parse(worksheet.Cells[row, 5].Value.ToString()),
gender = worksheet.Cells[row, 6].Value.ToString()
};
DB.personalDetails.Add(newPerson);
try { db.SaveChanges(); }
catch (Exception) { }
}
}
}
}
}//END OF IMPORT
ViewBag.Message = "Your file was successfully uploaded.";
return RedirectToAction("Index");
}
ViewBag.Message = "Error: Your file was not uploaded. Ensure you upload an excel workbook file.";
return View();
}
else
{
ViewBag.Message = "Error: Your file was not uploaded. Ensure you upload an excel workbook file.";
return View();
}
}
See Picture Error
Any help would be greatly appreciated mates.
you can do like this:
public bool readXLS(string FilePath)
{
FileInfo existingFile = new FileInfo(FilePath);
using (ExcelPackage package = new ExcelPackage(existingFile))
{
//get the first worksheet in the workbook
ExcelWorksheet worksheet = package.Workbook.Worksheets[1];
int colCount = worksheet.Dimension.End.Column; //get Column Count
int rowCount = worksheet.Dimension.End.Row; //get row count
string queryString = "INSERT INTO tableName VALUES"; //Here I am using "blind insert". You can specify the column names Blient inset is strongly not recommanded
string eachVal = "";
bool status;
for (int row = 1; row <= rowCount; row++)
{
queryString += "(";
for (int col = 1; col <= colCount; col++)
{
eachVal = worksheet.Cells[row, col].Value.ToString().Trim();
queryString += "'" + eachVal + "',";
}
queryString = queryString.Remove(queryString.Length - 1, 1); //removing last comma (,) from the string
if (row % 1000 == 0) //On every 1000 query will execute, as maximum of 1000 will be executed at a time.
{
queryString += ")";
status = this.runQuery(queryString); //executing query
if (status == false)
return status;
queryString = "INSERT INTO tableName VALUES";
}
else
{
queryString += "),";
}
}
queryString = queryString.Remove(queryString.Length - 1, 1); //removing last comma (,) from the string
status = this.runQuery(queryString); //executing query
return status;
}
}
Details: http://sforsuresh.in/read-data-excel-sheet-insert-database-table-c/
I stuck here please help, i'm new in flash, using adobe flash cs5.5
Here's the code :
import flash.net.URLLoaderDataFormat;
import flash.events.DataEvent;
stop();
alert_mc.visible = false;
alert_mc.ok_btn.visible = false;
var score:Number = 0;
var jawaban;
var nextQst:Number = 0;
var i:Number = 0;
a_btn.addEventListener(MouseEvent.CLICK,btna);
function btna(ev:MouseEvent):void
{
cekJawaban("a");
}
b_btn.addEventListener(MouseEvent.CLICK,btnb);
function btnb(ev:MouseEvent):void
{
cekJawaban("b");
}
c_btn.addEventListener(MouseEvent.CLICK,btnc);
function btnc(ev:MouseEvent):void
{
cekJawaban("c");
}
d_btn.addEventListener(MouseEvent.CLICK,btnd);
function btnd(ev:MouseEvent):void
{
cekJawaban("d");
}
var qvar_lv:URLLoader = new URLLoader();
qvar_lv.dataFormat = URLLoaderDataFormat.TEXT;// Step 1
qvar_lv.load(new URLRequest("DaftarSoal.txt")); // Step 2
qvar_lv.addEventListener(Event.COMPLETE, onComplete); // Step 3
function onComplete (event:Event):void // Step 4
{
var faq:String = event.target.data; // Step 5
trace("Success"+faq);
if (faq["Pertanyaan"+i] != undefined) {
no_txt.text = "PERTANYAAN KE-"+i;
soal_txt.text = faq["Pertanyaan"+i]; ja_txt.text = faq["a"+i];
jb_txt.text = faq["b"+i];
jc_txt.text = faq["c"+i];
jd_txt.text = faq["d"+i];
jawaban =faq["benar"+i];
} else {
gotoAndStop(3);
skor_txt.text = score+"";
teks_txt.text = "Score akhir kamu :";
}
}
function cekJawaban(val) {
alert_mc.visible = true;
if (val != jawaban) {
alert_mc.alert_txt.text = "jawaban kamu salah, score anda berkurang 50 points";
score = score-50;
alert_mc.ok_btn.addEventListener(MouseEvent.CLICK,btnok);
function btnok(ev:MouseEvent):void
{
nextQst = i+1;
alert_mc.visible = false;
}
} else {
score = score+100;
alert_mc.alert_txt.text = "jawaban kamu benar, score anda bertambah 100 points";
alert_mc.ok_btn.addEventListener(MouseEvent.CLICK,btnok2);
function btnok2(ev:MouseEvent):void
{
nextQst = i+1;
alert_mc.visible = false;
}
trace(score);
}
}
Error #1069:
Property Pertanyaan0 not found on String and there is no default value.
at revisi_fla::Symbol5_68/onComplete()
at flash.events::EventDispatcher/dispatchEventFunction()
at flash.events::EventDispatcher/dispatchEvent()
at flash.net::URLLoader/onComplete()
I have an array of files like this..
string[] unZippedFiles;
the idea is that I want to parse these files in paralle. As they are parsed a record gets placed on a concurrentbag. As record is getting placed I want to kick of the update function.
Here is what I am doing in my Main():
foreach(var file in unZippedFiles)
{ Parallel.Invoke
(
() => ImportFiles(file),
() => UpdateTest()
);
}
this is what the code of Update loooks like.
static void UpdateTest( )
{
Console.WriteLine("Updating/Inserting merchant information.");
while (!merchCollection.IsEmpty || producingRecords )
{
merchant x;
if (merchCollection.TryTake(out x))
{
UPDATE_MERCHANT(x.m_id, x.mInfo, x.month, x.year);
}
}
}
This is what the import code looks like. It's pretty much a giant string parser.
System.IO.StreamReader SR = new System.IO.StreamReader(fileName);
long COUNTER = 0;
StringBuilder contents = new StringBuilder( );
string M_ID = "";
string BOF_DELIMITER = "%%MS_SKEY_0000_000_PDF:";
string EOF_DELIMITER = "%%EOF";
try
{
record_count = 0;
producingRecords = true;
for (COUNTER = 0; COUNTER <= SR.BaseStream.Length - 1; COUNTER++)
{
if (SR.EndOfStream)
{
break;
}
contents.AppendLine(Strings.Trim(SR.ReadLine()));
contents.AppendLine(System.Environment.NewLine);
//contents += Strings.Trim(SR.ReadLine());
//contents += Strings.Chr(10);
if (contents.ToString().IndexOf((EOF_DELIMITER)) > -1)
{
if (contents.ToString().StartsWith(BOF_DELIMITER) & contents.ToString().IndexOf(EOF_DELIMITER) > -1)
{
string data = contents.ToString();
M_ID = data.Substring(data.IndexOf("_M") + 2, data.Substring(data.IndexOf("_M") + 2).IndexOf("_"));
Console.WriteLine("Merchant: " + M_ID);
merchant newmerch;
newmerch.m_id = M_ID;
newmerch.mInfo = data.Substring(0, (data.IndexOf(EOF_DELIMITER) + 5));
newmerch.month = DateTime.Now.AddMonths(-1).Month;
newmerch.year = DateTime.Now.AddMonths(-1).Year;
//Update(newmerch);
merchCollection.Add(newmerch);
}
contents.Clear();
//GC.Collect();
}
}
SR.Close();
// UpdateTest();
}
catch (Exception ex)
{
producingRecords = false;
}
finally
{
producingRecords = false;
}
}
the problem i am having is that the Update runs once and then the importfile function just takes over and does not yield to the update function. Any ideas on what am I doing wrong would be of great help.
Here's my stab at fixing your thread synchronisation. Note that I haven't changed any of the code from the functional standpoint (with the exception of taking out the catch - it's generally a bad idea; exceptions need to be propagated).
Forgive if something doesn't compile - I'm writing this based on incomplete snippets.
Main
foreach(var file in unZippedFiles)
{
using (var merchCollection = new BlockingCollection<merchant>())
{
Parallel.Invoke
(
() => ImportFiles(file, merchCollection),
() => UpdateTest(merchCollection)
);
}
}
Update
private void UpdateTest(BlockingCollection<merchant> merchCollection)
{
Console.WriteLine("Updating/Inserting merchant information.");
foreach (merchant x in merchCollection.GetConsumingEnumerable())
{
UPDATE_MERCHANT(x.m_id, x.mInfo, x.month, x.year);
}
}
Import
Don't forget to pass in merchCollection as a parameter - it should not be static.
System.IO.StreamReader SR = new System.IO.StreamReader(fileName);
long COUNTER = 0;
StringBuilder contents = new StringBuilder( );
string M_ID = "";
string BOF_DELIMITER = "%%MS_SKEY_0000_000_PDF:";
string EOF_DELIMITER = "%%EOF";
try
{
record_count = 0;
for (COUNTER = 0; COUNTER <= SR.BaseStream.Length - 1; COUNTER++)
{
if (SR.EndOfStream)
{
break;
}
contents.AppendLine(Strings.Trim(SR.ReadLine()));
contents.AppendLine(System.Environment.NewLine);
//contents += Strings.Trim(SR.ReadLine());
//contents += Strings.Chr(10);
if (contents.ToString().IndexOf((EOF_DELIMITER)) > -1)
{
if (contents.ToString().StartsWith(BOF_DELIMITER) & contents.ToString().IndexOf(EOF_DELIMITER) > -1)
{
string data = contents.ToString();
M_ID = data.Substring(data.IndexOf("_M") + 2, data.Substring(data.IndexOf("_M") + 2).IndexOf("_"));
Console.WriteLine("Merchant: " + M_ID);
merchant newmerch;
newmerch.m_id = M_ID;
newmerch.mInfo = data.Substring(0, (data.IndexOf(EOF_DELIMITER) + 5));
newmerch.month = DateTime.Now.AddMonths(-1).Month;
newmerch.year = DateTime.Now.AddMonths(-1).Year;
//Update(newmerch);
merchCollection.Add(newmerch);
}
contents.Clear();
//GC.Collect();
}
}
SR.Close();
// UpdateTest();
}
finally
{
merchCollection.CompleteAdding();
}
}
Am using Fck editor to write content. Am storing the text as versions in db. I want to highlight those changes in versions when loading the text in FCK Editor.
How to compare the text....
How to show any text that has been deleted in strike through mode.
Please help me/...
Try google's diff-patch algorithm http://code.google.com/p/google-diff-match-patch/
Take both previous and current version of the text and store it into two parameters. Pass the two parameters to the following function.
function diffString(o, n) {
o = o.replace(/<[^<|>]+?>| /gi, '');
n = n.replace(/<[^<|>]+?>| /gi, '');
var out = diff(o == "" ? [] : o.split(/\s+/), n == "" ? [] : n.split(/\s+/));
var str = "";
var oSpace = o.match(/\s+/g);
if (oSpace == null) {
oSpace = ["\n"];
} else {
oSpace.push("\n");
}
var nSpace = n.match(/\s+/g);
if (nSpace == null) {
nSpace = ["\n"];
} else {
nSpace.push("\n");
}
if (out.n.length == 0) {
for (var i = 0; i < out.o.length; i++) {
str += '<span style="background-color:#F00;"><del>' + escape(out.o[i]) + oSpace[i] + "</del></span>";
}
} else {
if (out.n[0].text == null) {
for (n = 0; n < out.o.length && out.o[n].text == null; n++) {
str += '<span style="background-color:#F00;"><del>' + escape(out.o[n]) + oSpace[n] + "</del></span>";
}
}
for (var i = 0; i < out.n.length; i++) {
if (out.n[i].text == null) {
str += '<span style="background-color:#0C0;"><ins>' + escape(out.n[i]) + nSpace[i] + "</ins></span>";
} else {
var pre = "";
for (n = out.n[i].row + 1; n < out.o.length && out.o[n].text == null; n++) {
pre += '<span style="background-color:#F00;"><del>' + escape(out.o[n]) + oSpace[n] + "</del></span>";
}
str += " " + out.n[i].text + nSpace[i] + pre;
}
}
}
return str;
}
this returns an html in which new text is marked green and deleted text as red + striked out.