How to create 64 bit inode? - linux

I need to test 64bit version of file IO APIs (open, create stat etc. ). In this process I need to create a file which has a 64 bit inode, so that the internal 64 bit data structures/variables are tested and so the APIs. How do I create a 64 bit inode ?
I have written a script where in I am trying to create a nested array of directories with 1024 files in each directories. The script takes huge amount of time to execute and terminates abruptly. I am not able to proceed, is there any other way to achieve it?

You could simulate any inode number you want by using FUSE.
Look at the hello_ll.c example that comes with FUSE. It creates a filesystem with a single file that has inode number 2. You could modify that file pretty easily to create files with whatever inode number you want.
A quick test with 0x10000000FFFFFFL does this:
$ stat fuse/hello
File: `fuse/hello'
Size: 13 Blocks: 0 IO Block: 4096 regular file
Device: 11h/17d Inode: 4503599644147711 Links: 1
Access: (0444/-r--r--r--) Uid: ( 0/ root) Gid: ( 0/ root)
Other than FUSE, I know of no practical way of forcing an inode number on "real" filesystems.
Here's a minimal patch used to produce that:
--- hello_ll.c.orig 2011-11-14 13:22:19.000000000 +0100
+++ hello_ll.c 2011-11-14 13:20:27.000000000 +0100
## -9,6 +9,7 ##
*/
#define FUSE_USE_VERSION 26
+#define MYINO 0x10000000FFFFFFL
#include <fuse_lowlevel.h>
#include <stdio.h>
## -31,7 +32,7 ##
stbuf->st_nlink = 2;
break;
- case 2:
+ case MYINO:
stbuf->st_mode = S_IFREG | 0444;
stbuf->st_nlink = 1;
stbuf->st_size = strlen(hello_str);
## -65,7 +66,7 ##
fuse_reply_err(req, ENOENT);
else {
memset(&e, 0, sizeof(e));
- e.ino = 2;
+ e.ino = MYINO;
e.attr_timeout = 1.0;
e.entry_timeout = 1.0;
hello_stat(e.ino, &e.attr);
## -117,7 +118,7 ##
memset(&b, 0, sizeof(b));
dirbuf_add(req, &b, ".", 1);
dirbuf_add(req, &b, "..", 1);
- dirbuf_add(req, &b, hello_name, 2);
+ dirbuf_add(req, &b, hello_name, MYINO);
reply_buf_limited(req, b.p, b.size, off, size);
free(b.p);
}
## -126,7 +127,7 ##
static void hello_ll_open(fuse_req_t req, fuse_ino_t ino,
struct fuse_file_info *fi)
{
- if (ino != 2)
+ if (ino != MYINO)
fuse_reply_err(req, EISDIR);
else if ((fi->flags & 3) != O_RDONLY)
fuse_reply_err(req, EACCES);
## -139,7 +140,7 ##
{
(void) fi;
- assert(ino == 2);
+ assert(ino == MYINO);
reply_buf_limited(req, hello_str, strlen(hello_str), off, size);
}

You would have to create 4294967296 files or directories.
In order to do so, you have to prepare your file system to have space for this. Depending on which file system you use, it may or may not be possible. (I just tried to do so with a ext4 file system, and it didn't work.)

You could use a systemtap script to simply bump up the inode number returned by a stat call.
On ext4, something like:
probe kernel.statement("ext4_getattr#fs/ext4/inode.c+21")
{
$stat->ino = $stat->ino + 4294967295;
}
probe begin { log("starting probe") }
would do the trick (you might have to adjust the "21" offset, if ext4_getattr is different in your tree).

Related

Get the serial number of the volume udf cd / dvd disk?

I'm writing a program in linux which counts the serial number (xxxx-xxxx) of the volume of the CD in Windows 7. My program correctly determines the serial number of the volume on disks with the filesystems iso9660 and joilet. But how to define a disk volume sniffer with a file system udf? Can someone tell me ....
ps if anyone does not understand I'm talking about the serial number of this kind https://extra-torrent.jimdo.com/2016/01/23/hard-disk-volume-serial-number-change/
#include <QCoreApplication>
#include <stdio.h>
#include <sys/ioctl.h>
#include <linux/cdrom.h>
#include <string.h>
#include <szi/szimac.h>
#include <qfile.h>
#include <iostream>
#include <QDir>
#include <unistd.h>
#define SEC_SIZE 2048
#define VD_N 16
#define VD_TYPE_SUPP 2
#define VD_TYPE_END 255
#define ESC_IDX 88
#define ESC_LEN 3
#define ESC_UCS2L1 "%/#"
#define ESC_UCS2L2 "%/C"
#define ESC_UCS2L3 "%/E"
using namespace std;
int cdid(unsigned char pvd[SEC_SIZE])
{
unsigned char part[4] = {0};
int i;
for(i = 0; i < SEC_SIZE; i += 4)
{
part[3] += pvd[i + 0];
part[2] += pvd[i + 1];
part[1] += pvd[i + 2];
part[0] += pvd[i + 3];
}
return (part[3] << 24) + (part[2] << 16) + (part[1] << 8) + part[0];
}
int main(int argc, char *argv[])
{
FILE *in;
unsigned char buf[SEC_SIZE];
struct cdrom_multisession msinfo;
long session_start;
int id;
QString home=QString(getenv("HOME"))+QString("/chteniestorm");
QFile file(home);
ustr="/dev/sr0";
in = fopen(ustr.toLocal8Bit().data(), "rb");
if(in == NULL)
{
if (file.open(QIODevice::WriteOnly))
{
file.write("sernom=1");
file.close();
}
cout<<"netdiska"<<endl;
return 0;
}
/* Get session info */
msinfo.addr_format = CDROM_LBA;
if(ioctl(fileno(in), CDROMMULTISESSION, &msinfo) != 0)
{
fprintf(stderr, "WARNING: Can't get multisession info\n");
perror(NULL);
session_start = 0;
}
else
{
session_start = msinfo.addr.lba;
}
fseek(in, 0, SEEK_SET); //to the begining
/* Seek to primary volume descriptor */
if(fseek(in, (session_start + VD_N) * SEC_SIZE, SEEK_SET) != 0)
{
if (file.open(QIODevice::WriteOnly))
{
file.write("sernom=2");
file.close();
}
fclose(in);
return 0;
}
/* Read descriptor */
if(fread(buf, 1, SEC_SIZE, in) != SEC_SIZE)
{
if (file.open(QIODevice::WriteOnly))
{
file.write("sernom=3");
file.close();
}
fclose(in);
return 0;
}
/* Caclculate disc id */
id = cdid(buf);
/* Search for Joliet extension */
while(buf[0] != VD_TYPE_END)
{
/* Read descriptor */
if(fread(buf, 1, SEC_SIZE, in) != SEC_SIZE)
{
perror(NULL);
return 0;
}
if(buf[0] == VD_TYPE_SUPP
&& (memcmp(buf + ESC_IDX, ESC_UCS2L1, ESC_LEN) == 0
|| memcmp(buf + ESC_IDX, ESC_UCS2L2, ESC_LEN) == 0
|| memcmp(buf + ESC_IDX, ESC_UCS2L3, ESC_LEN) == 0)
)
{
/* Joliet found */
id = cdid(buf);
}
}
fclose(in);
}
It looks like this question was asked on more places [1], [2], [3], [4] but nowhere was answered yet. So I will do it here.
In some of those posts people decoded serial number generation algorithm. It is just checksum which you already have found and put into your cdid() function. Same checksum algorithm is used for both ISO9660 and UDF filesystems on Windows. You have already figured out from which ISO9660 structures is that checksum calculated.
So your question remain just for UDF filesystem. For UDF filesystem on Windows that checksum is calculated from the 512 bytes long File Set Descriptor (FSD) structure. I would suggest you to read OSTA UDF specification how to locale that FSD on UDF disc.
Basically for plain UDF which do not use Virtual Allocation Table (VAT), Sparing Table or Metadata Partition, location of the FSD is stored in Logical Volume Descriptor (LVD) structure, in field LogicalVolumeContentsUse (it is of type long_ad). LVD is stored in the Volume Descriptor Sequence (VDS). VDS's location is stored in Anchor Volume Descriptor Pointer (AVDP), in field MainVolumeDescriptorSequenceExtent. AVDP itself is located at sector 256 of medium. Optical media have sector size 2048 bytes and common hard disk 512 bytes.
For UDF with VAT (e.g. on CD-R/DVD-R/BD-R), Sparing Table (e.g. on CD-RW/DVD-RW) or Metadata Partition (e.g. on Blu-ray), it is much more complicated. You need to look into Virtual, Sparable or Metadata Partition to figure out how to translate logical location of the FSD to physical location of media.
In udftools project starting with version 2.0, there is a new tool udfinfo which provides various information about UDF filesystem. It shows also that Windows specific Volume Serial Number from your question under winserialnum key. Note that udfinfo cannot read FSD from UDF filesystem with VAT or Metadata yet.

Invalid compressed data--format violated?

I want to extract data from xxx.tar.gz file by using tar -zxvf command, but something wrong occurs to me, here's the detail:
suse11-configserver:/home/webapp/wiki # tar -zxvf dokuwiki.20151010.tar.gz
./dokuwiki/
./dokuwiki/._.htaccess.dist
./dokuwiki/.htaccess.dist
./dokuwiki/bin/
./dokuwiki/conf/
./dokuwiki/._COPYING
./dokuwiki/COPYING
tar: Jump to the next head
gzip: stdin: invalid compressed data--format violated
tar: Child returned status 1
tar: Error is not recoverable: exiting now
But this command tar -zxvf dokuwiki.20151010.tar.gz goes well in the MacOS x System, I can not figure out the reason.
Your command is correct. But it seems the file is corrupted.
It's easy to tell, when some files are correctly extracted (for example ./dokuwiki/.htaccess.dist), but not the rest.
Recreate the dokuwiki.20151010.tar.gz file, and make sure it doesn't report errors while doing so.
If you downloaded the file from somewhere, verify the checksum, or at least the file size.
The bottomline is, either the file was incorrectly created or downloaded.
The command you have should work fine with a .tar.gz file.
Alternative Location of Gzip's fixgz Utility
In case you can no longer find fixgz on gzip.org's website, here is a link to a version available on archive.org: https://web.archive.org/web/20180624175352/http://www.gzip.org/fixgz.zip.
Source Code for fixgz Utility
Also, in case that disappears as well, below is the source code for the fixgz utility:
/* fixgz attempts to fix a binary file transferred in ascii mode by
* removing each extra CR when it followed by LF.
* usage: fixgz bad.gz fixed.gz
* Copyright 1998 Jean-loup Gailly <jloup#gzip.org>
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the author be held liable for any damages
* arising from the use of this software.
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely.
*/
#include <stdio.h>
int main(argc, argv)
int argc;
char **argv;
{
int c1, c2; /* input bytes */
FILE *in; /* corrupted input file */
FILE *out; /* fixed output file */
if (argc <= 2) {
fprintf(stderr, "usage: fixgz bad.gz fixed.gz\n");
exit(1);
}
in = fopen(argv[1], "rb");
if (in == NULL) {
fprintf(stderr, "fixgz: cannot open %s\n", argv[1]);
exit(1);
}
out = fopen(argv[2], "wb");
if (in == NULL) {
fprintf(stderr, "fixgz: cannot create %s\n", argv[2]);
exit(1);
}
c1 = fgetc(in);
while ((c2 = fgetc(in)) != EOF) {
if (c1 != '\r' || c2 != '\n') {
fputc(c1, out);
}
c1 = c2;
}
if (c1 != EOF) {
fputc(c1, out);
}
exit(0);
return 0; /* avoid warning */
}
Gzip has a prospective fix for this error in their FAQ. The provided utility didn't help in my case, but it's possible it would fix your archive. According to gzip:
If you have transferred a file in ASCII mode and you no longer have access to the original, you can try the program fixgz to remove the extra CR (carriage return) bytes inserted by the transfer. A Windows 9x/NT/2000/ME/XP binary is here. But there is absolutely no guarantee that this will actually fix your file. Conclusion: never transfer binary files in ASCII mode.

Linux file descriptor table and vmalloc

I see that the Linux kernel uses vmalloc to allocate memory for fdtable when it's bigger than a certain threshold. I would like to know when this happens and have some more clear information.
static void *alloc_fdmem(size_t size)
{
/*
* Very large allocations can stress page reclaim, so fall back to
* vmalloc() if the allocation size will be considered "large" by the VM.
*/
if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
void *data = kmalloc(size, GFP_KERNEL|__GFP_NOWARN);
if (data != NULL)
return data;
}
return vmalloc(size);
}
alloc_fdmem is called from alloc_fdtable and the last function is called from expand_fdtable
I wrote this code to print the size.
#include <stdio.h>
#define PAGE_ALLOC_COSTLY_ORDER 3
#define PAGE_SIZE 4096
int main(){
printf("\t%d\n", PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER);
}
Output
./printo
32768
So, how many files does it take for the kernel to switch to using vmalloc to allocate fdtable?
So PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER is 32768
This is called like:
data = alloc_fdmem(nr * sizeof(struct file *));
i.e. it's used to store struct file pointers.
If your pointers are 4 bytes, it happens when your have 32768/4 = 8192 open files, if your pointers are 8 bytes, it happens at 4096 open files.

Geting numbers of files in specific directory in Linux

Is there anyway to get the total number of files in a specific directory not iterating readdir(3)?
I mean only direct members of a specific directory.
It seems that the only way to get the number of files is calling readdir(3) repeatedly until it returns zero.
Is there any other way to get the number in O(1)? I need a solution that works in Linux.
Thanks.
scandir() example, requires dirent.h:
struct dirent **namelist;
int n=scandir(".", &namelist, 0, alphasort); // "." == current directory.
if (n < 0)
{
perror("scandir");
exit(1);
}
printf("files found = %d\n", n);
free(namelist);
I don't think it is possible in O(1).
Just think about inode structure. There's no clue for this.
But if it's OK to get number of files in a filesystem,
you can use statvfs(2).
#include <sys/vfs.h> /* or <sys/statfs.h> */
int statfs(const char *path, struct statfs *buf);
struct statfs {
__SWORD_TYPE f_type; /* type of file system (see below) */
__SWORD_TYPE f_bsize; /* optimal transfer block size */
fsblkcnt_t f_blocks; /* total data blocks in file system */
fsblkcnt_t f_bfree; /* free blocks in fs */
fsblkcnt_t f_bavail; /* free blocks available to
unprivileged user */
fsfilcnt_t f_files; /* total file nodes in file system */
fsfilcnt_t f_ffree; /* free file nodes in fs */
fsid_t f_fsid; /* file system id */
__SWORD_TYPE f_namelen; /* maximum length of filenames */
__SWORD_TYPE f_frsize; /* fragment size (since Linux 2.6) */
__SWORD_TYPE f_spare[5];
};
You can easily get number of files via f_files - f_ffree.
BTW, This is very interesting question. so I voted it up.
In shell script it's damn simple
ll directory_path | wc -l

Tools to reduce risk regarding password security and HDD slack space

Down at the bottom of this essay is a comment about a spooky way to beat passwords. Scan the entire HDD of a user including dead space, swap space etc, and just try everything that looks like it might be a password.
The question: part 1, are there any tools around (A live CD for instance) that will scan an unmounted file system and zero everything that can be? (Note I'm not trying to find passwords)
This would include:
Slack space that is not part of any file
Unused parts of the last block used by a file
Swap space
Hibernation files
Dead space inside of some types of binary files (like .DOC)
The tool (aside from the last case) would not modify anything that can be detected via the file system API. I'm not looking for a block device find/replace but rather something that just scrubs everything that isn't part of a file.
part 2, How practical would such a program be? How hard would it be to write? How common is it for file formats to contain uninitialized data?
One (risky and costly) way to do this would be to use a file system aware backup tool (one that only copies the actual data) to back up the whole disk, wipe it clean and then restore it.
I don't understand your first question (do you want to modify the file system? Why? Isn't this dead space exactly where you want to look?)
Anyway, here's an example of such a tool:
#include <stdio.h>
#include <alloca.h>
#include <string.h>
#include <ctype.h>
/* Number of bytes we read at once, >2*maxlen */
#define BUFSIZE (1024*1024)
/* Replace this with a function that tests the passwort consisting of the first len bytes of pw */
int testPassword(const char* pw, int len) {
/*char* buf = alloca(len+1);
memcpy(buf, pw,len);
buf[len] = '\0';
printf("Testing %s\n", buf);*/
int rightLen = strlen("secret");
return len == rightLen && memcmp(pw, "secret", len) == 0;
}
int main(int argc, char* argv[]) {
int minlen = 5; /* We know the password is at least 5 characters long */
int maxlen = 7; /* ... and at most 7. Modify to find longer ones */
int avlen = 0; /* available length - The number of bytes we already tested and think could belong to a password */
int i;
char* curstart;
char* curp;
FILE* f;
size_t bytes_read;
char* buf = alloca(BUFSIZE+maxlen);
if (argc != 2) {
printf ("Usage: %s disk-file\n", argv[0]);
return 1;
}
f = fopen(argv[1], "rb");
if (f == NULL) {
printf("Couldn't open %s\n", argv[1]);
return 2;
}
for(;;) {
/* Copy the rest of the buffer to the front */
memcpy(buf, buf+BUFSIZE, maxlen);
bytes_read = fread(buf+maxlen, 1, BUFSIZE, f);
if (bytes_read == 0) {
/* Read the whole file */
break;
}
for (curstart = buf;curstart < buf+bytes_read;) {
for (curp = curstart+avlen;curp < curstart + maxlen;curp++) {
/* Let's assume the password just contains letters and digits. Use isprint() otherwise. */
if (!isalnum(*curp)) {
curstart = curp + 1;
break;
}
}
avlen = curp - curstart;
if (avlen < minlen) {
/* Nothing to test here, move along */
curstart = curp+1;
avlen = 0;
continue;
}
for (i = minlen;i <= avlen;i++) {
if (testPassword(curstart, i)) {
char* found = alloca(i+1);
memcpy(found, curstart, i);
found[i] = '\0';
printf("Found password: %s\n", found);
}
}
avlen--;
curstart++;
}
}
fclose(f);
return 0;
}
Installation:
Start a Linux Live CD
Copy the program to the file hddpass.c in your home directory
Open a terminal and type the following
su || sudo -s # Makes you root so that you can access the HDD
apt-get install -y gcc # Install gcc
This works only on Debian/Ubuntu et al, check your system documentation for others
gcc -o hddpass hddpass.c # Compile.
./hddpass /dev/YOURDISK # The disk is usually sda, hda on older systems
Look at the output
Test (copy to console, as root):
gcc -o hddpass hddpass.c
</dev/zero head -c 10000000 >testdisk # Create an empty 10MB file
mkfs.ext2 -F testdisk # Create a file system
rm -rf mountpoint; mkdir -p mountpoint
mount -o loop testdisk mountpoint # needs root rights
</dev/urandom head -c 5000000 >mountpoint/f # Write stuff to the disk
echo asddsasecretads >> mountpoint/f # Write password in our pagefile
# On some file systems, you could even remove the file.
umount testdisk
./hdpass testdisk # prints secret
Test it yourself on an Ubuntu Live CD:
# Start a console and type:
wget http://phihag.de/2009/so/hddpass-testscript.sh
sh hddpass-testscript.sh
Therefore, it's relatively easy. As I found out myself, ext2 (the file system I used) overwrites deleted files. However, I'm pretty sure some file systems don't. Same goes for the pagefile.
How common is it for file formats to contain uninitialized data?
Less and less common, I would've thought. The classic "offender" is older versions of MS office applications that (essentially) did a memory dump to disk as its "quicksave" format. No serialisation, no selection of what to dump and a memory allocator that doesn't zero newly allocated memory pages. That lead to not only juicy things from previous versions of the document (so the user could use undo), but also juicy snippets from other applications.
How hard would it be to write?
Something that clears out unallocated disk blocks shouldn't be that hard. It'd need to run either off-line or as a kernel module, so as to not interfer with normal file-system operations, but most file systems have an "allocated"/"not allocated" structure that is fairly straight-forward to parse. Swap is harder, but as long as you're OK with having it cleared on boot (or shutdown), it's not too tricky. Clearing out the tail block is trickier, definitely not something I'd want to try to do on-line, but it shouldn't be TOO hard to make it work for off-line cleaning.
How practical would such a program be?
Depends on your threat model, really. I'd say that on one end, it'd not give you much at all, but on the other end, it's a definite help to keep information out of the wrong hands. But I can't give a hard and fast answer,
Well, if I was going to code it for a boot CD, I'd do something like this:
File is 101 bytes but takes up a 4096-byte cluster.
Copy the file "A" to "B" which has nulls added to the end.
Delete "A" and overwrite it's (now unused) cluster.
Create "A" again and use the contents of "B" without the tail (remember the length).
Delete "B" and overwrite it.
Not very efficient, and would need a tweak to make sure you don't try to copy the first (and therefor full) clusters in a file. Otherwise, you'll run into slowness and failure if there's not enough free space.
There's tools that do this efficiently that are open source?

Resources