The code works very good, and creates the data, but in Linux it says the the data has 0 observations.
It has to search in library the files with specific name (and date in name).
%let path=library.name;
%let fname=name.starts.with;
data fnames;
length
fref $8
fname $200
datestr $17
;
format fdate yymmdd10.;
rc = filename(fref,"&path.");
did = dopen(fref);
if did ne 0
then do;
do i = 1 to dnum(did);
fname = dread(did,i);
if index(fname,"&fname.")
then do;
datestr = scan(scan(fname,-2,"."),-1,"_");
date = input(substr(datestr,1,8),yymmdd8.);
fdate = substr(datestr,1,8);
output;
end;
end;
rc = dclose(did);
end;
rc = filename(fref);
keep fname fdate;
run;
proc sort data=fnames;
by descending fdate;
run;
data _null_;
set fnames (obs=1);
call symputx("fdate",fdate,"g");
run;
How can I run the code in Linux? Why is it not read the data correctly?
Thank you.
Related
If i have an Excel "Covid19_report &todaysdate. &time..xlsx" file in a folder called F:\A
and I want to put this exact file into folder F:\B but duplicated 6 times like this:
"1_Covid19_report &todaysdate. &time..xlsx"
"2_Covid19_report &todaysdate. &time..xlsx"
"3_Covid19_report &todaysdate. &time..xlsx"
"4_Covid19_report &todaysdate. &time..xlsx"
"5_Covid19_report &todaysdate. &time..xlsx"
"6_Covid19_report &todaysdate. &time..xlsx"
is there a macro to do that?
The SAS function FCOPY will be helpful.
Copying binary files requires a fileref be created with options such as RECFM=N and LRECL=<filesize>
Example:
Create a sample Excel file and copy it 6 times to target files having an index prefix in their filenames.
* Create sample Excel file for copying;
ods noresults;
ods excel file='c:\temp\prices.xlsx';
proc print data=sashelp.stocks;
run;
ods excel close;
ods results;
* Copy a source file 6 times to a target file with an index number prefix name pattern;
data _null_;
length source_ref target_ref $8;
* determine name part from full pathname;
source_file = 'c:\temp\prices.xlsx';
source_name = scan(source_file,-1,'\');
* determine source file size for efficient options;
* NOTE: FCOPY only works on files < 1G in size;
rc = filename(source_ref, source_file);
fid = fopen(source_ref);
bytes = finfo(fid,'File size (bytes)');
fid = fclose(fid);
rc = filename(source_ref);
options = 'recfm=n lrecl='||trim(bytes);
* prepare SOURCE fileref;
source_ref = '';
rc = filename(source_ref, source_file, 'DISK', options);
* duplicate in same folder (via transtrn);
do index = 1 to 6;
* compute target file name and corresponding full path;
target_name = cats(index,"_",source_name);
target_file = transtrn(source_file, trim(source_name), target_name);
* create temporary TARGETR fileref (due to ref value passed in being blank);
target_ref = '';
if 0 = filename (target_ref, target_file, 'DISK', options) then do;
rc = fcopy ('source', target_ref);
rc = filename(target_ref); * clear TARGET fileref;
end;
end;
stop;
run;
i tried doing this, but do not work:
%let todaysDate = %sysfunc(today(), yymmdd10.); %put &todaysDate;
data _null_; ftime = intnx('hour',time(),0,'b'); cftime =
compress(put(ftime,hhmm.),":"); call symputx("ftime",cftime); run;
%put &=ftime.;
%let path = F:\A;
%let file = COVID-19 Report &todaysdate. &ftime.;
filename source "&path.\&file.";
data _null_; length target_ref $8;
do index = 1 to 6;
target_file = cats("&path.",index,"_&file.");
target_ref = '';
* create temporary fileref (due to initial ref value being blank);
if 0 = filename (target_ref, target_file) then do;
rc = fcopy ('source', target_ref);
rc = filename(target_ref);
end; end; stop; run;
I'm in need of optimizing import of .xls files to matlab due to xlsread being very time consuming with large amount of files. Current xlsread script as follows:
scriptName = mfilename('fullpath');
[currentpath, filename, fileextension]= fileparts(scriptName);
xlsnames = dir(fullfile(currentpath,'*.xls'));
xlscount = length(xlsnames);
xlsimportdata = zeros(7,6,xlscount);
for k = 1:xlscount
xlsimport = xlsread(xlsnames(k).name,'D31:I37');
xlsimportdata(:,1:size(xlsimport,2),k) = xlsimport;
end
I have close to 10k files per week that needs processing and with approx. 2sec per file processed on my current workstation, it comes in at about 5½ hours.
I have read that ActiveX can be used for this purpose however that is far beyond my current programming skills and have not been able to find a solution elsewhere. Any help on how to make this would be appreciated.
If it is simple to perform with ActiveX (or other proposed method), I would also be interested in data on cells D5 and G3, which I am currently grabbing from 'xlsnames(k,1).name' and 'xlsnames(k,1).date'
EDIT: updated to reflect the solution
% Get path to .m script
scriptName = mfilename('fullpath');
[currentpath, filename, fileextension]= fileparts(scriptName);
% Generate list of .xls file data
xlsnames = dir(fullfile(currentpath,'*.xls'));
xlscount = length(xlsnames);
SampleInfo = cell(xlscount,2);
xlsimportdata = cell(7,6,xlscount);
% Define xls data ranges to import
SampleID = 'G3';
SampleRuntime = 'D5';
data_range = 'D31:I37';
% Initiate progression bar
h = waitbar(0,'Initiating import...');
% Start actxserver
exl = actxserver('excel.application');
exlWkbk = exl.Workbooks;
for k = 1:xlscount
% Restart actxserver every 100 loops due limited system memory
if mod (k,100) == 0
exl.Quit
exl = actxserver('excel.application');
exlWkbk = exl.Workbooks;
end
exlFile = exlWkbk.Open([dname filesep xlsnames(k).name]);
exlSheet1 = exlFile.Sheets.Item('Page 0');
rngObj1 = exlSheet1.Range(SampleID);
xlsimport_ID = rngObj1.Value;
rngObj2 = exlSheet1.Range(SampleRuntime);
xlsimport_Runtime = rngObj2.Value;
rngObj3 = exlSheet1.Range(data_range);
xlsimport_data = rngObj3.Value;
SampleInfo(k,1) = {xlsimport_ID};
SampleInfo(k,2) = {xlsimport_Runtime};
xlsimportdata(:,:,k) = xlsimport_data;
% Progression bar updater
progress = round((k / xlscount) * 100);
importtext = sprintf('Importing %d of %d', k, xlscount);
waitbar(progress/100,h,sprintf(importtext));
disp(['Import progress: ' num2str(k) '/' num2str(xlscount)]);
end
%close actxserver
exl.Quit
% Close progression bar
close(h)
Give this a try. I am not an ActiveX Excel guru by any means. However, this works for me for my small amount of test XLS files (3). I never close the exlWkbk so I don't know if memory usage is building or if it automatically cleaned up when descoped after the next is opened in its place ... so use at your own risk. I am seeing an almost 2.5x speed increase which seems promising.
>> timeit(#getSomeXLS)
ans =
1.8641
>> timeit(#getSomeXLS_old)
ans =
4.6192
Please leave some feedback if this work on large number of Excel sheets because I am curious how it goes.
function xlsimportdata = getSomeXLS()
scriptName = mfilename('fullpath');
[currentpath, filename, fileextension]= fileparts(scriptName);
xlsnames = dir(fullfile(currentpath,'*.xls'));
xlscount = length(xlsnames);
xlsimportdata = zeros(7,6,xlscount);
exl = actxserver('excel.application');
exlWkbk = exl.Workbooks;
dat_range = 'D31:I37';
for k = 1:xlscount
exlFile = exlWkbk.Open([currentpath filesep xlsnames(k).name]);
exlSheet1 = exlFile.Sheets.Item('Sheet1'); %Whatever your sheet is called.
rngObj = exlSheet1.Range(dat_range);
xlsimport = cell2mat(rngObj.Value);
xlsimportdata(:,:,k) = xlsimport;
end
exl.Quit
I have a file (termino.txt) that is all filled in the following format :
pay the bill
2015-08-30T13:22:53.108Z
Go to the doctor
2015-09-30T13:22:53.108Z
....
All the even lines are of the form RFC 3339 timestamp. What I need is to compare today's date with these dates the file to see if they are the same. I'm trying this:
local function verifica(evt)
local nome= ''
local dia = ''
local turn = 1
local data = os.date("%x")
local file = io.open("termino.txt", "r")
while true do
nome = dia
line = file:read()
dia = line
if (turn %2 == 0) then
> Here I need to compare "data" with "dia" that will receive string with RFC 3339 timestamp format.
end
turn ++
end
end
I need help to make this comparison! Thanks
local dia = '2015-10-6T13:22:53.108Z'
-- parse date info from the RFC 3339 timestamp
local year, month, day = dia:match('(%d+)-(%d+)-(%d+)')
-- get today's date from Lua, in table format
local today = os.date('*t')
-- compare
if tonumber(year) == today.year
and tonumber(month) == today.month
and tonumber(day) == today.day then
-- the dates match
end
The data in my Excel files is supposed to be contentious (index in the first column). But some data is missing in the file. For example, # 5, and 6 are missing between $ 4 and 7. My purpose are (1) identify the file with missing data and (2) if data is missing insert rows to make it continuous. Can anyone tell me how to add in rows in the existing data? Using xlswrite I can only add in rows at the end of the file or replace some rows.
EDIT 1:
I have another set of file in which the index is not so direct. The first 3 columns are described below (as shown in the Excel file):
Column 1：Year: 2003 (read as number in matlab)
Column 2：Date: 1-Sep (read as text in matlab)
Column 3：Time: 1:00 (1:00 read as number 0.04167 and 2:00 read as 0.0833, not sure how it works)
Then the way to tell if it is continuous will be quite complicate since there will be different years, months, and days. Could you give some hint on this?
Basically you need to read the entire data, preferably in raw(cell) format, add the missing rows(with respect to the indices) and write back.
Based on your question, this code might work -
% NOTE: We are assuming that the indexing starts with 1
% Read data from input excel file with missing indices
[num,txt,raw] = xlsread('input.xls');
% Error-checking
if (size(num,1)-num(end,1)~=0)
disp('At least one index is missing!');
end
% Expand data such that all indices are covered.
data1=NaN(num(end,1),size(raw,2));
data1(:,1) = 1:num(end,1);
data1=num2cell(data1);
k1=1;
for k = 1:num(end,1)
if(num(k1,1)==k)
data1(k,:)= raw(k1,:);
k1 = k1+1;
end
end
% Write data
xlswrite('output.xls',data1);
EDIT 1:
In view of your new requirements, additional code is added next.
Please note few things about this code -
The code adds data for every year and not from a specific month, date and time to another specific month, date and time. If you wish to achieve that, please edit the associated
function - 'create_comp_sheet'.
It saves an intermediate file named - 'proper_base_data.xls', which maybe deleted at the end of the code.
%% MAIN CODE - CODE1.M
INPUT_FILENAME = 'input.xls'; % Excel file that has some missing year,date and time info
OUTPUT_FILENAME = 'output.xls'; % Excel file that has data from the input file along with all the missing year,date and time info
%% Base data
start_year=2003;
end_year=2005;
proper_base_data = create_comp_sheet(start_year,end_year);
xlswrite('proper_base_data.xls',proper_base_data);
[num,txt,raw] = xlsread('proper_base_data.xls');
base_data=cell(size(num,1),1);
for row_ID = 1:size(num,1)
base_data(row_ID) = {strcat(num2str(cell2mat(raw(row_ID,1))),'-', cell2mat(raw(row_ID,2)),'-',num2str(round(24*cell2mat(raw(row_ID,3)))))};
end
%% Input data
[num,txt,raw] = xlsread(INPUT_FILENAME);
input_data=cell(size(num,1),1);
for row_ID = 1:size(num,1)
input_data(row_ID) = {strcat(num2str(cell2mat(raw(row_ID,1))),'-', cell2mat(raw(row_ID,2)),'-',num2str(round(24*cell2mat(raw(row_ID,3)))))};
end
%% Setup final data
final_data = num2cell(NaN(size(proper_base_data,1),size(raw,2)));
final_data(:,1:3) = proper_base_data;
for k1=1:size(input_data,1)
for k2=1:size(base_data,1)
if strcmp(cell2mat(base_data(k2)),cell2mat(input_data(k1)))
final_data(k2,4:end) = raw(k1,4:end);
end
end
end
%% Write final data to excel
xlswrite(OUTPUT_FILENAME,final_data);
Associated function -
function data1 = create_comp_sheet(start_year,end_year)
months_string = {'Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'};
date_count = [31 28 31 30 31 30 31 31 30 31 30 31];
num_hours = 24;
data1=[];
for year_ID = start_year:end_year
for month_ID = 1:numel(months_string)
days_per_month = date_count(month_ID);
if rem(year_ID,4)==0 && month_ID ==2
days_per_month = days_per_month+1;
end
for date_ID = 1:days_per_month
year = repmat({num2str(year_ID)},[num_hours 1]);
date = repmat({strcat(num2str(date_ID),'-',char(months_string(month_ID)))},[num_hours 1]);
time=cell(num_hours,1);
for k = 1:num_hours
time(k) = {strcat(num2str(k),':00')};
end
data1 = [data1 ; [year date time]];
end
end
end
return;
Hope this saves all your troubles!
I'm having issues with the foloowing code and I can't seem to find much information to help me sort it out. I'm trying to write some filenames from a directory to a datset, then create a zip file of those files. It works fine until I reach the data step with the infile statement. I received the following error...
ERROR: No logical assign for filename DIRLIST.
Here is my code...
%macro get_filenames(location);
filename _dir_ "%bquote(&location.)";
data filenames(keep=fname);
handle=dopen( '_dir_' );
if handle > 0 then do;
count=dnum(handle);
do i=1 to count;
fname=dread(handle,i);
output filenames;
end;
end;
rc=dclose(handle);
run;
filename _dir_ clear;
%mend;
%get_filenames(c:\temp\);
data dirlist;
set filenames;
where fname like 'scra%.txt';
run;
ods package(testfile) open nopf;
data _null_;
infile dirlist pad lrecl=80;
input #1 filename $80.;
call execute
(catx
(' ',
'ods package(testfile)',
'add file=',
quote('c:\temp\' || trim(filename)),
';'
)
);
run;
ods package(testfile) publish archive
properties(archive_name='testfile.zip'
archive_path='c:\temp\' );
ods package(testfile) close;
I think SAS is complaining about the following in your code:
data _null_;
infile dirlist pad lrecl=80;
Infile in this context is expecting a FILENAME reference the kind you have at filename _dir_ "%bquote(&location.)";
It does not understand that you want to use the dataset dirlist.
Replace the above code snippet with the following:
data _null_;
SET dirlist;