fallocate() command equivalent in OS X? - linux

Is there a fallocate() equivalent in OS X?
I would like to aggregate all of those equivalent in OS X questions into some doc/table or whatever for everyone. Anybody knows something familiar?

What about using:
mkfile -n 1m test.tmp
It's not the same command but serves the same purpose.
Note that fallocate uses decimal multipliers, whereas mkfile uses binary multipliers.
mkfile man

fallocate() doesn't exist on OSX. You can "fake" it though; Mozilla fakes it in their FileUtils class. See this file:
http://hg.mozilla.org/mozilla-central/file/3d846420a907/xpcom/glue/FileUtils.cpp#l61
Here's the code, in case that link goes stale:
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is Mozilla code.
*
* The Initial Developer of the Original Code is
* Mozilla Foundation.
* Portions created by the Initial Developer are Copyright (C) 2010
* the Initial Developer. All Rights Reserved.
*
* Contributor(s):
* Taras Glek <tglek#mozilla.com>
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
#if defined(XP_UNIX)
#include <fcntl.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#elif defined(XP_WIN)
#include <windows.h>
#endif
#include "nscore.h"
#include "private/pprio.h"
#include "mozilla/FileUtils.h"
bool
mozilla::fallocate(PRFileDesc *aFD, PRInt64 aLength)
{
#if defined(HAVE_POSIX_FALLOCATE)
return posix_fallocate(PR_FileDesc2NativeHandle(aFD), 0, aLength) == 0;
#elif defined(XP_WIN)
return PR_Seek64(aFD, aLength, PR_SEEK_SET) == aLength
&& 0 != SetEndOfFile((HANDLE)PR_FileDesc2NativeHandle(aFD));
#elif defined(XP_MACOSX)
int fd = PR_FileDesc2NativeHandle(aFD);
fstore_t store = {F_ALLOCATECONTIG, F_PEOFPOSMODE, 0, aLength};
// Try to get a continous chunk of disk space
int ret = fcntl(fd, F_PREALLOCATE, &store);
if(-1 == ret){
// OK, perhaps we are too fragmented, allocate non-continuous
store.fst_flags = F_ALLOCATEALL;
ret = fcntl(fd, F_PREALLOCATE, &store);
if (-1 == ret)
return false;
}
return 0 == ftruncate(fd, aLength);
#elif defined(XP_UNIX)
// The following is copied from fcntlSizeHint in sqlite
/* If the OS does not have posix_fallocate(), fake it. First use
** ftruncate() to set the file size, then write a single byte to
** the last byte in each block within the extended region. This
** is the same technique used by glibc to implement posix_fallocate()
** on systems that do not have a real fallocate() system call.
*/
struct stat buf;
int fd = PR_FileDesc2NativeHandle(aFD);
if (fstat(fd, &buf))
return false;
if (buf.st_size >= aLength)
return false;
const int nBlk = buf.st_blksize;
if (!nBlk)
return false;
if (ftruncate(fd, aLength))
return false;
int nWrite; // Return value from write()
PRInt64 iWrite = ((buf.st_size + 2 * nBlk - 1) / nBlk) * nBlk - 1; // Next offset to write to
do {
nWrite = 0;
if (PR_Seek64(aFD, iWrite, PR_SEEK_SET) == iWrite)
nWrite = PR_Write(aFD, "", 1);
iWrite += nBlk;
} while (nWrite == 1 && iWrite < aLength);
return nWrite == 1;
#endif
return false;
}

For those wanting to create fake data files for testing, mkfile is pretty elegant. An alternative is to use dd:
dd if=/dev/zero of=zfile count=1024 bs=1024
As you can see with od -b zfile, it's full of zeros. If you want random data (which you may want for testing workflows with data compression, for example), then use "/dev/random" in place of "/dev/zero":
dd if=/dev/random of=randfile count=1024 bs=1024

Related

I was running a simple ns3 simulation (using leo-model) and i get this question

I created a simple simulation, (same like first.cc) ,use the channel that leo-model gave.
Since i set Unlimited transmission rate, there is no propagation delay, transmission delay and processing delay.But simulation still use 0.008s and 0.001s.I dont konw how this delay (0.008s and 0.001s) come from?
And what is the meaning of m_currentTs=next.ky.m_ts? Since that, the delay 0.008s occuerd.
here are my codes:
/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation;
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include "ns3/core-module.h"
#include "ns3/network-module.h"
#include "ns3/internet-module.h"
#include "ns3/point-to-point-module.h"
#include "ns3/applications-module.h"
#include "ns3/applications-module.h"
#include "ns3/isl-helper.h"
#include "ns3/leo-module.h"
// Default Network Topology
//
// 10.1.1.0
// n0 -------------- n1
// point-to-point
//
using namespace ns3;
NS_LOG_COMPONENT_DEFINE ("FirstLEOScriptExample");
int
main (int argc, char *argv[])
{
Address remote;
CommandLine cmd (__FILE__);
cmd.Parse (argc, argv);
Time::SetResolution (Time::NS);
LogComponentEnable ("UdpEchoClientApplication", LOG_LEVEL_INFO);
LogComponentEnable ("UdpEchoServerApplication", LOG_LEVEL_INFO);
NodeContainer nodes; // NodeContainer
NodeContainer satellites;
NodeContainer stations;
satellites.Create(1);
stations.Create(1);
LeoChannelHelper utCh;
utCh.SetGndDeviceAttribute("DataRate", StringValue("99999999Mbps"));
utCh.SetSatDeviceAttribute("DataRate", StringValue("99999999Mbps"));
NetDeviceContainer utNet = utCh.Install(satellites, stations);
// NodeContainer
InternetStackHelper stack;
stack.Install (satellites);
stack.Install(stations);
Ipv4AddressHelper address;
address.SetBase ("10.1.1.0", "255.255.255.0");
address.Assign(utNet);
// ------------------------------------------------------------------
UdpEchoServerHelper echoServer (9);
ApplicationContainer serverApps = echoServer.Install (stations.Get (0));
serverApps.Start (Seconds (1.0));
serverApps.Stop (Seconds (10.0));
// ------------------------------------------------------------------
Ptr<Node> server = stations.Get (0);
remote = server->GetObject<Ipv4> ()->GetAddress (1, 0).GetLocal ();
UdpEchoClientHelper echoClient (remote, 9);
echoClient.SetAttribute ("MaxPackets", UintegerValue (1));
echoClient.SetAttribute ("Interval", TimeValue (Seconds (0.0)));
echoClient.SetAttribute ("PacketSize", UintegerValue (1024));
ApplicationContainer clientApps = echoClient.Install (satellites.Get (0));
clientApps.Start (Seconds (2.0));
clientApps.Stop (Seconds (10.0));
// ------------------------------------------------------------------
Simulator::Run ();
Simulator::Destroy ();
return 0;
}
i tryed gdb but cant understand the reason.

Invalid compressed data--format violated?

I want to extract data from xxx.tar.gz file by using tar -zxvf command, but something wrong occurs to me, here's the detail:
suse11-configserver:/home/webapp/wiki # tar -zxvf dokuwiki.20151010.tar.gz
./dokuwiki/
./dokuwiki/._.htaccess.dist
./dokuwiki/.htaccess.dist
./dokuwiki/bin/
./dokuwiki/conf/
./dokuwiki/._COPYING
./dokuwiki/COPYING
tar: Jump to the next head
gzip: stdin: invalid compressed data--format violated
tar: Child returned status 1
tar: Error is not recoverable: exiting now
But this command tar -zxvf dokuwiki.20151010.tar.gz goes well in the MacOS x System, I can not figure out the reason.
Your command is correct. But it seems the file is corrupted.
It's easy to tell, when some files are correctly extracted (for example ./dokuwiki/.htaccess.dist), but not the rest.
Recreate the dokuwiki.20151010.tar.gz file, and make sure it doesn't report errors while doing so.
If you downloaded the file from somewhere, verify the checksum, or at least the file size.
The bottomline is, either the file was incorrectly created or downloaded.
The command you have should work fine with a .tar.gz file.
Alternative Location of Gzip's fixgz Utility
In case you can no longer find fixgz on gzip.org's website, here is a link to a version available on archive.org: https://web.archive.org/web/20180624175352/http://www.gzip.org/fixgz.zip.
Source Code for fixgz Utility
Also, in case that disappears as well, below is the source code for the fixgz utility:
/* fixgz attempts to fix a binary file transferred in ascii mode by
* removing each extra CR when it followed by LF.
* usage: fixgz bad.gz fixed.gz
* Copyright 1998 Jean-loup Gailly <jloup#gzip.org>
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the author be held liable for any damages
* arising from the use of this software.
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely.
*/
#include <stdio.h>
int main(argc, argv)
int argc;
char **argv;
{
int c1, c2; /* input bytes */
FILE *in; /* corrupted input file */
FILE *out; /* fixed output file */
if (argc <= 2) {
fprintf(stderr, "usage: fixgz bad.gz fixed.gz\n");
exit(1);
}
in = fopen(argv[1], "rb");
if (in == NULL) {
fprintf(stderr, "fixgz: cannot open %s\n", argv[1]);
exit(1);
}
out = fopen(argv[2], "wb");
if (in == NULL) {
fprintf(stderr, "fixgz: cannot create %s\n", argv[2]);
exit(1);
}
c1 = fgetc(in);
while ((c2 = fgetc(in)) != EOF) {
if (c1 != '\r' || c2 != '\n') {
fputc(c1, out);
}
c1 = c2;
}
if (c1 != EOF) {
fputc(c1, out);
}
exit(0);
return 0; /* avoid warning */
}
Gzip has a prospective fix for this error in their FAQ. The provided utility didn't help in my case, but it's possible it would fix your archive. According to gzip:
If you have transferred a file in ASCII mode and you no longer have access to the original, you can try the program fixgz to remove the extra CR (carriage return) bytes inserted by the transfer. A Windows 9x/NT/2000/ME/XP binary is here. But there is absolutely no guarantee that this will actually fix your file. Conclusion: never transfer binary files in ASCII mode.

shm_open() fails with EINVAL when creating shared memory in subdirectory of /dev/shm

I have a GNU/Linux application with uses a number of shared memory objects. It could, potentially, be run a number of times on the same system. To keep things tidy, I first create a directory in /dev/shm for each of the set of shared memory objects.
The problem is that on newer GNU/Linux distributions, I no longer seem to be able create these in a sub-directory of /dev/shm.
The following is a minimal C program with illustrates what I'm talking about:
/*****************************************************************************
* shm_minimal.c
*
* Test shm_open()
*
* Expect to create shared memory file in:
* /dev/shm/
* └── my_dir
*    └── shm_name
*
* NOTE: Only visible on filesystem during execution. I try to be nice, and
* clean up after myself.
*
* Compile with:
* $ gcc -lrt shm_minimal.c -o shm_minimal
*
******************************************************************************/
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
int main(int argc, const char* argv[]) {
int shm_fd = -1;
char* shm_dir = "/dev/shm/my_dir";
char* shm_file = "/my_dir/shm_name"; /* does NOT work */
//char* shm_file = "/my_dir_shm_name"; /* works */
// Create directory in /dev/shm
mkdir(shm_dir, 0777);
// make shared memory segment
shm_fd = shm_open(shm_file, O_RDWR | O_CREAT, 0600);
if (-1 == shm_fd) {
switch (errno) {
case EINVAL:
/* Confirmed on:
* kernel v3.14, GNU libc v2.19 (ArchLinux)
* kernel v3.13, GNU libc v2.19 (Ubuntu 14.04 Beta 2)
*/
perror("FAIL - EINVAL");
return 1;
default:
printf("Some other problem not being tested\n");
return 2;
}
} else {
/* Confirmed on:
* kernel v3.8, GNU libc v2.17 (Mint 15)
* kernel v3.2, GNU libc v2.15 (Xubuntu 12.04 LTS)
* kernel v3.1, GNU libc v2.13 (Debian 6.0)
* kernel v2.6.32, GNU libc v2.12 (RHEL 6.4)
*/
printf("Success !!!\n");
}
// clean up
close(shm_fd);
shm_unlink(shm_file);
rmdir(shm_dir);
return 0;
}
/* vi: set ts=2 sw=2 ai expandtab:
*/
When I run this program on a fairly new distribution, the call to shm_open() returns -1, and errno is set to EINVAL. However, when I run on something a little older, it creates the shared memory object in /dev/shm/my_dir as expected.
For the larger application, the solution is simple. I can use a common prefix instead of a directory.
If you could help enlighten me to this apparent change in behavior it would be very helpful. I suspect someone else out there might be trying to do something similar.
So it turns out the issue stems from how GNU libc validates the shared memory name. Specifically, the shared memory object MUST now be at the root of the shmfs mount point.
This was changed in glibc git commit b20de2c3d9 as the result of bug BZ #16274.
Specifically, the change is the line:
if (name[0] == '\0' || namelen > NAME_MAX || strchr (name, '/') != NULL)
Which now disallows '/' from anywhere in the filename (not counting leading '/')
If you have a third party tool that was broken by this shm_open change, a brilliant coworker found a workaround : preload a library that overrides the shm_open call and swaps slashes for underscores. It does the same for shm_unlink as well, so the application can properly free shared memory when needed.
deslash_shm.cc :
#include <dlfcn.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <algorithm>
#include <string>
// function used in place of the standard shm_open() function
extern "C" int shm_open(const char *name, int oflag, mode_t mode)
{
// keep a function pointer to the real shm_open() function
static int (*real_open)(const char *, int, mode_t) = NULL;
// the first time in, ask the dynamic linker to find the real shm_open() function
if (!real_open) real_open = (int (*)(const char *, int, mode_t)) dlsym(RTLD_NEXT,"shm_open");
// take the name we were given and replace all slashes with underscores instead
std::string n = name;
std::replace(n.begin(), n.end(), '/', '_');
// call the real open function with the patched path name
return real_open(n.c_str(), oflag, mode);
}
// function used in place of the standard shm_unlink() function
extern "C" int shm_unlink(const char *name)
{
// keep a function pointer to the real shm_unlink() function
static int (*real_unlink)(const char *) = NULL;
// the first time in, ask the dynamic linker to find the real shm_unlink() function
if (!real_unlink) real_unlink = (int (*)(const char *)) dlsym(RTLD_NEXT, "shm_unlink");
// take the name we were given and replace all slashes with underscores instead
std::string n = name;
std::replace(n.begin(), n.end(), '/', '_');
// call the real unlink function with the patched path name
return real_unlink(n.c_str());
}
To compile this file:
c++ -fPIC -shared -o deslash_shm.so deslash_shm.cc -ldl
And preload it before starting a process that tries to use non-standard slash characters in shm_open:
in bash:
export LD_PRELOAD=/path/to/deslash_shm.so
in tcsh:
setenv LD_PRELOAD /path/to/deslash_shm.so

How to create 64 bit inode?

I need to test 64bit version of file IO APIs (open, create stat etc. ). In this process I need to create a file which has a 64 bit inode, so that the internal 64 bit data structures/variables are tested and so the APIs. How do I create a 64 bit inode ?
I have written a script where in I am trying to create a nested array of directories with 1024 files in each directories. The script takes huge amount of time to execute and terminates abruptly. I am not able to proceed, is there any other way to achieve it?
You could simulate any inode number you want by using FUSE.
Look at the hello_ll.c example that comes with FUSE. It creates a filesystem with a single file that has inode number 2. You could modify that file pretty easily to create files with whatever inode number you want.
A quick test with 0x10000000FFFFFFL does this:
$ stat fuse/hello
File: `fuse/hello'
Size: 13 Blocks: 0 IO Block: 4096 regular file
Device: 11h/17d Inode: 4503599644147711 Links: 1
Access: (0444/-r--r--r--) Uid: ( 0/ root) Gid: ( 0/ root)
Other than FUSE, I know of no practical way of forcing an inode number on "real" filesystems.
Here's a minimal patch used to produce that:
--- hello_ll.c.orig 2011-11-14 13:22:19.000000000 +0100
+++ hello_ll.c 2011-11-14 13:20:27.000000000 +0100
## -9,6 +9,7 ##
*/
#define FUSE_USE_VERSION 26
+#define MYINO 0x10000000FFFFFFL
#include <fuse_lowlevel.h>
#include <stdio.h>
## -31,7 +32,7 ##
stbuf->st_nlink = 2;
break;
- case 2:
+ case MYINO:
stbuf->st_mode = S_IFREG | 0444;
stbuf->st_nlink = 1;
stbuf->st_size = strlen(hello_str);
## -65,7 +66,7 ##
fuse_reply_err(req, ENOENT);
else {
memset(&e, 0, sizeof(e));
- e.ino = 2;
+ e.ino = MYINO;
e.attr_timeout = 1.0;
e.entry_timeout = 1.0;
hello_stat(e.ino, &e.attr);
## -117,7 +118,7 ##
memset(&b, 0, sizeof(b));
dirbuf_add(req, &b, ".", 1);
dirbuf_add(req, &b, "..", 1);
- dirbuf_add(req, &b, hello_name, 2);
+ dirbuf_add(req, &b, hello_name, MYINO);
reply_buf_limited(req, b.p, b.size, off, size);
free(b.p);
}
## -126,7 +127,7 ##
static void hello_ll_open(fuse_req_t req, fuse_ino_t ino,
struct fuse_file_info *fi)
{
- if (ino != 2)
+ if (ino != MYINO)
fuse_reply_err(req, EISDIR);
else if ((fi->flags & 3) != O_RDONLY)
fuse_reply_err(req, EACCES);
## -139,7 +140,7 ##
{
(void) fi;
- assert(ino == 2);
+ assert(ino == MYINO);
reply_buf_limited(req, hello_str, strlen(hello_str), off, size);
}
You would have to create 4294967296 files or directories.
In order to do so, you have to prepare your file system to have space for this. Depending on which file system you use, it may or may not be possible. (I just tried to do so with a ext4 file system, and it didn't work.)
You could use a systemtap script to simply bump up the inode number returned by a stat call.
On ext4, something like:
probe kernel.statement("ext4_getattr#fs/ext4/inode.c+21")
{
$stat->ino = $stat->ino + 4294967295;
}
probe begin { log("starting probe") }
would do the trick (you might have to adjust the "21" offset, if ext4_getattr is different in your tree).

What is the return value of sched_find_first_bit if it doesn't find anything?

The kernel is 2.4.
On a side note, does anybody knows a good place where I can search for that kind of information? Searching Google for function definitions is frustrating.
If you plan on spending any significant time searching through or understanding the Linux kernel, I recommend downloading a copy and using Cscope.
Using Cscope on large projects (example: the Linux kernel)
I found the following in a copy of the Linux kernel 2.4.18.
The key seems to be the comment before this last piece of code below. It appears that the return value of sched_find_first_bit is undefined if no bit is set.
From linux-2.4/include/linux/sched.h:185
/*
* The maximum RT priority is configurable. If the resulting
* bitmap is 160-bits , we can use a hand-coded routine which
* is optimal. Otherwise, we fall back on a generic routine for
* finding the first set bit from an arbitrarily-sized bitmap.
*/
#if MAX_PRIO 127
#define sched_find_first_bit(map) _sched_find_first_bit(map)
#else
#define sched_find_first_bit(map) find_first_bit(map, MAX_PRIO)
#endif
From linux-2.4/include/asm-i386/bitops.h:303
/**
* find_first_bit - find the first set bit in a memory region
* #addr: The address to start the search at
* #size: The maximum size to search
*
* Returns the bit-number of the first set bit, not the number of the byte
* containing a bit.
*/
static __inline__ int find_first_bit(void * addr, unsigned size)
{
int d0, d1;
int res;
/* This looks at memory. Mark it volatile to tell gcc not to move it around */
__asm__ __volatile__(
"xorl %%eax,%%eax\n\t"
"repe; scasl\n\t"
"jz 1f\n\t"
"leal -4(%%edi),%%edi\n\t"
"bsfl (%%edi),%%eax\n"
"1:\tsubl %%ebx,%%edi\n\t"
"shll $3,%%edi\n\t"
"addl %%edi,%%eax"
:"=a" (res), "=&c" (d0), "=&D" (d1)
:"1" ((size + 31) >> 5), "2" (addr), "b" (addr));
return res;
}
From linux-2.4/include/asm-i386/bitops.h:425
/*
* Every architecture must define this function. It's the fastest
* way of searching a 140-bit bitmap where the first 100 bits are
* unlikely to be set. It's guaranteed that at least one of the 140
* bits is cleared.
*/
static inline int _sched_find_first_bit(unsigned long *b)
{
if (unlikely(b[0]))
return __ffs(b[0]);
if (unlikely(b[1]))
return __ffs(b[1]) + 32;
if (unlikely(b[2]))
return __ffs(b[2]) + 64;
if (b[3])
return __ffs(b[3]) + 96;
return __ffs(b[4]) + 128;
}
From linux-2.4/include/asm-i386/bitops.h:409
/**
* __ffs - find first bit in word.
* #word: The word to search
*
* Undefined if no bit exists, so code should check against 0 first.
*/
static __inline__ unsigned long __ffs(unsigned long word)
{
__asm__("bsfl %1,%0"
:"=r" (word)
:"rm" (word));
return word;
}

Resources