Problems with netlink socket, kernel opps 0002 - linux

I try to send some data from user mode to my kernel module. But when it send something, it triggers a kernel error:
Bug:unable to handle kernel paging request ad ffff88022f168bc0
IP: [<...>]build_skb+0xf5/0x1c0
#include "socket.hpp"
using namespace std;
KernelSocket::KernelSocket(DataLink * dl){
this->dl = dl;
this->sock_fd = socket(PF_NETLINK, SOCK_RAW, NETLINK_USER);
if(sock_fd<0) {
printf("unable to create socket%d\n", sock_fd);
exit(-1);
}
memset(&src_addr, 0, sizeof(src_addr));
src_addr.nl_family = AF_NETLINK;
src_addr.nl_pid = getpid();
//src_addr.nl_groups = 0;
bind(sock_fd, (struct sockaddr*)&src_addr, sizeof(src_addr));
memset(&dest_addr, 0, sizeof(dest_addr));
memset(&dest_addr, 0, sizeof(dest_addr));
dest_addr.nl_family = AF_NETLINK;
dest_addr.nl_pid = 0;
dest_addr.nl_groups = 0;
nlh = (struct nlmsghdr *)malloc(NLMSG_SPACE(MAX_PAYLOAD));
memset(nlh,0,NLMSG_SPACE(MAX_PAYLOAD));
nlh->nlmsg_len = NLMSG_SPACE(MAX_PAYLOAD);
nlh->nlmsg_pid = getpid();
nlh->nlmsg_flags = 0;
reportLocalAddress();
}
void KernelSocket::sendToKernel(string data){
int length = data.length();
char c;
c = (length%0xff);
data.insert(0, 1, c);
c = (length/0xff);
data.insert(0, 1, c);
nlh->nlmsg_len = NLMSG_SPACE(MAX_PAYLOAD);
nlh->nlmsg_pid = getpid();
nlh->nlmsg_flags = 0;
memcpy((NLMSG_DATA(nlh)), data.c_str(), length+2);
iov.iov_base = (void *)nlh;
iov.iov_len = nlh->nlmsg_len;
//iov.iov_len = length;
msg.msg_name = (void *)&dest_addr;
msg.msg_namelen = sizeof(dest_addr);
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
sendmsg(sock_fd,&msg,0);
}
void KernelSocket::reportLocalAddress(){
int length = 6;
string data = dl->localAddress.toString();
data.insert(0,1, 0xff);
data.insert(0,1, 0xff);
nlh->nlmsg_len = NLMSG_SPACE(MAX_PAYLOAD);
nlh->nlmsg_pid = getpid();
nlh->nlmsg_flags = 0;
memcpy((NLMSG_DATA(nlh)), data.c_str(), length+2);
iov.iov_base = (void *)nlh;
iov.iov_len = nlh->nlmsg_len;
msg.msg_name = (void *)&dest_addr;
msg.msg_namelen = sizeof(dest_addr);
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
sendmsg(sock_fd,&msg,0);
printf("send mac address to kernel\n");
}
void * kernelSocketWorkLoop(void * kernelSocket){
int length,ret;
KernelSocket * ks = (KernelSocket *) kernelSocket;
while(1){
ret = recvmsg(ks->sock_fd, &(ks->msg), MSG_DONTWAIT);
if(ret>=0){
string c;
c.assign((char *)NLMSG_DATA(ks->nlh));
length = c[0]*0xff+c[1];
ks->dl->packetDownBuffer.push(c);
}
sleep(1);
}
}
I am using this period code now. Anybody have opinion on this problem? Thanks~
Here is the function in the kernel module that I use to receive message.
static void ss_recv(struct sk_buff *skb){
struct nlmsghdr *nlh;
int len;
char * c;
struct uwmodem_priv* priv = NULL;
priv = netdev_priv(dev);
printk("in ss_recv\n");
nlh = (struct nlmsghdr *)skb->data;
if(nlh->nlmsg_pid>0){
priv->daemon_pid = nlh->nlmsg_pid;
len = nlh->nlmsg_len - NLMSG_HDRLEN;
c = kmalloc(len, GFP_ATOMIC);
if(c==NULL){
printk("Fail to allocate c\n");
}
memcpy(c, nlmsg_data(nlh), len);
if(c[0]==0xff && c[1]==0xff){
printk("Set MacAddress(%d) \n", nlh->nlmsg_pid);
//for mac address
memcpy(dev->dev_addr, c+2, ETH_ALEN);
}else{
len = c[0]*0xff+c[1];
//printk("Kernel Length:[%d][%d][%d]\n", c[0], c[1], c[2]);
printk("Kernel received from (%d)[%d]: \n", nlh->nlmsg_pid,len);
uwmodem_rx(dev, c+2, len);
}
kfree(c);
}
//printk("Exit ss_recv\n");
}

Related

Why is the network latency so high in Linux socket hardware time stamping?

I am trying to obtain the packet traveling latency in hardware timestamp between 2 computers connected by 1 Ethernet cable. However the result for obtained delay is 11.5 micro-seconds, which is much higher than my expectation like several nano-seconds.
Design
The way I tried to obtain the latency is demonstrated in the following graph as delay = ((t3 - t0) - (t2 - t1)) / 2
client server
| ping |
t0|-------------->|t1
| |
| pong |
t3|<--------------|t2
| |
v v
Here t0/t2 are Linux hardware sending time recorded by NIC, and t1/t3 are hardware receiving timestamp. The detailed information about linux socket hardware time can be found here: https://docs.kernel.org/networking/timestamping.html. As my understand, t0/t2 are time that NIC transmit the first byte to wire, t1/t3 are time that NIC receive the first byte from wire. Thus the delay that I calculated is purely the propagation delay which should be within several nanoseconds. In addition to prove my assumption, I tried to change the packet size in 256/512/1400 bytes, the results have no change and are all in 11.5 microseconds. Is my understand correct on those hardware timestamp and delay computation?
Implementation
The 2 computers I am using are Dell OptiPlex-7090 on Ubuntu 20.04, the NIC version is Ethernet Connection (14) I219-LM.
The completed code for testing can be found here: https://github.com/ChuanyuXue/Hardware-RTT. It records both hardware timestamp and software timestamp.
utils.c:
void die(char *s)
{
perror(s);
exit(1);
}
int setup_receiver(int fd, int port)
{
struct sockaddr_in addr;
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_ANY);
addr.sin_port = htons(port);
if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) == -1)
{
die("bind()");
}
int val = SOF_TIMESTAMPING_RX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE;
if (setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPING, &val, sizeof(val)) == -1)
{
die("setsockopt() HW receiving");
}
int enable = 1;
if (setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPNS, &enable, sizeof(enable)) < 0)
{
die("setsockopt() SW receiving");
}
return 0;
}
int setup_sender(int fd)
{
int timestamp_flags = SOF_TIMESTAMPING_TX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE;
if (setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPING, &timestamp_flags, sizeof(timestamp_flags)) < 0)
{
die("setsockopt() HW sending");
}
int enable = 1;
if (setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPNS, &enable, sizeof(enable)))
{
die("setsockopt() SW receiving");
}
}
int setup_adapter(int fd, char *dev_name)
{
struct hwtstamp_config hwts_config;
struct ifreq ifr;
memset(&hwts_config, 0, sizeof(hwts_config));
hwts_config.tx_type = HWTSTAMP_TX_ON;
hwts_config.rx_filter = HWTSTAMP_FILTER_ALL;
memset(&ifr, 0, sizeof(ifr));
snprintf(ifr.ifr_name, IFNAMSIZ, "%s", dev_name);
ifr.ifr_data = (void *)&hwts_config;
if (ioctl(fd, SIOCSHWTSTAMP, &ifr) == -1)
{
die("ioctl()");
}
}
void send_single(int fd, char *address, int port)
{
/*
Send one message
*/
struct sockaddr_in si_server;
memset(&si_server, 0, sizeof(si_server));
si_server.sin_family = AF_INET;
si_server.sin_port = htons(port);
if (inet_aton(address, &si_server.sin_addr) == 0)
{
die("inet_aton()");
}
char buffer[BUFFER_LEN];
struct iovec iov = (struct iovec){.iov_base = buffer, .iov_len = BUFFER_LEN};
struct msghdr msg = (struct msghdr){.msg_name = &si_server,
.msg_namelen = sizeof si_server,
.msg_iov = &iov,
.msg_iovlen = 1};
ssize_t send_len = sendmsg(fd, &msg, 0);
if (send_len < 0)
{
printf("[!] Error sendmsg()");
}
// -------------- obtain the loopback packet
char data[BUFFER_LEN], control[BUFFER_LEN];
struct iovec entry;
struct sockaddr_in from_addr;
int res;
memset(&msg, 0, sizeof(msg));
msg.msg_iov = &entry;
msg.msg_iovlen = 1;
entry.iov_base = data;
entry.iov_len = sizeof(data);
msg.msg_name = (caddr_t)&from_addr;
msg.msg_namelen = sizeof(from_addr);
msg.msg_control = &control;
msg.msg_controllen = sizeof(control);
// wait until get the loopback
while (recvmsg(fd, &msg, MSG_ERRQUEUE) < 0)
{
}
// encode the returned packet
struct cmsghdr *cmsg;
for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL;
cmsg = CMSG_NXTHDR(&msg, cmsg))
{
if (cmsg->cmsg_level == SOL_SOCKET &&
cmsg->cmsg_type == SCM_TIMESTAMPING)
{
struct timespec *ts =
(struct timespec *)CMSG_DATA(cmsg);
printf("HD-SEND TIMESTAMP %ld.%09ld\n", ts[2].tv_sec, ts[2].tv_nsec);
}
if (cmsg->cmsg_level == SOL_SOCKET &&
cmsg->cmsg_type == SO_TIMESTAMPNS)
{
struct timespec *ts =
(struct timespec *)CMSG_DATA(cmsg);
printf("SW-SEND TIMESTAMP %ld.%09ld\n", ts->tv_sec, ts->tv_nsec);
}
}
}
void recv_single(int fd)
{
char data[BUFFER_LEN], ctrl[BUFFER_LEN];
struct msghdr msg;
struct iovec iov;
ssize_t len;
struct cmsghdr *cmsg;
memset(&msg, 0, sizeof(msg));
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
msg.msg_control = ctrl;
msg.msg_controllen = sizeof(ctrl);
iov.iov_base = data;
iov.iov_len = sizeof(data);
struct timespec start;
if (recvmsg(fd, &msg, 0) < 0)
{
printf("[!] Error recvmsg()");
}
for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL;
cmsg = CMSG_NXTHDR(&msg, cmsg))
{
if (cmsg->cmsg_level == SOL_SOCKET &&
cmsg->cmsg_type == SCM_TIMESTAMPING)
{
struct timespec *ts =
(struct timespec *)CMSG_DATA(cmsg);
printf("HW-RECV TIMESTAMP %ld.%09ld\n", ts[2].tv_sec, ts[2].tv_nsec);
}
if (cmsg->cmsg_level == SOL_SOCKET &&
cmsg->cmsg_type == SCM_TIMESTAMPNS)
{
struct timespec *ts =
(struct timespec *)CMSG_DATA(cmsg);
printf("SW-RECV TIMESTAMP %ld.%09ld\n", ts->tv_sec, ts->tv_nsec);
}
}
}
client.c
int main(int argc, char *argv[])
{
const char *address = "192.168.0.23";
const int port = 54321;
int fd_in = socket(AF_INET, SOCK_DGRAM, 0);
int fd_out = socket(AF_INET, SOCK_DGRAM, 0);
setup_adapter(fd_in, "eth0");
setup_adapter(fd_out, "eth0");
setup_sender(fd_out);
setup_receiver(fd_in, port);
int count = 0;
while (1)
{
printf("[ ---- Iter-%5d ----------------------------- ]\n", count++);
send_single(fd_out, (char *)address, port);
recv_single(fd_in);
usleep(10000);
}
}
server.c
int main(int argc, char *argv[])
{
const char *address = "192.168.0.22";
const int port = 54321;
int fd_in = socket(AF_INET, SOCK_DGRAM, 0);
int fd_out = socket(AF_INET, SOCK_DGRAM, 0);
setup_adapter(fd_in, "eth0");
setup_adapter(fd_out, "eth0");
setup_sender(fd_out);
setup_receiver(fd_in, port);
int count = 0;
while (1)
{
printf("[ ---- Iter-%5d ----------------------------- ]\n", count++);
recv_single(fd_in);
usleep(10000);
send_single(fd_out, (char *)address, port);
}
}
From the results, I can see the hardware timestamp is much more stable and has very low jitter compared with software timestamp(I am sorry my account is not allowed to embed picture into question):
https://github.com/ChuanyuXue/Hardware-RTT/blob/main/0301_exp_hwtime/01.png
https://github.com/ChuanyuXue/Hardware-RTT/blob/main/0301_exp_hwtime/02.png
https://github.com/ChuanyuXue/Hardware-RTT/blob/main/0301_exp_hwtime/03.png
https://github.com/ChuanyuXue/Hardware-RTT/blob/main/0301_exp_hwtime/04.png
https://github.com/ChuanyuXue/Hardware-RTT/blob/main/0301_exp_hwtime/06.png
Above results make me feel the timestamp I obtained is correct as the HW latency has smaller delay and very low jitter, but there are some issues I neglect when I compute delay = ((t3 - t0) - (t2 - t1)) / 2. Is there anyone who can let me know why or test my code on other environment?

Using select for packet sockets

Is it possible to use select for packet sockets? I mean whether its necessary for the socket to be connection-based in order to use select function on it properly?
I'm seeing that the behavior of a socket which I got it by the following socket function call:
int socket_fd = socket(PF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
is not as I expect. For example, I see that a ping packet which is 60 bytes long when received and read into a 20 bytes length buffer, it waits about a second between each recv function call. I used recvfrom and it didn't help. For this, I ask whether it's correct to use select for a packet socket?
Update
I'm going to include the code to discuss about it:
int MakeBridge(const char *if1, const char *if2)
{
int sock[2];
sock[0] = OpenSocket(if1);
sock[1] = OpenSocket(if2);
int activity;
char buf[20];
const int numSocks = 2;
int nfds = sock[0];
for (int i = 1; i < numSocks; i++)
if (sock[i] > nfds)
nfds = sock[i];
nfds++;
while (true)
{
FD_ZERO(&readfds);
for (int i = 0; i < numSocks; i++)
FD_SET(sock[i], &readfds);
activity = select(nfds, &readfds, NULL, NULL, NULL);
if (activity == -1) // sockets closed
break;
for (int i = 0; i < numSocks; i++)
if (FD_ISSET(_sock[i], &readfds))
{
int len;
CHECK(ioctl(_sock[i], FIONREAD, &len) != -1, "%s", strerror(errno));
printf("socket %d is set\n", i);
printf("total bytes available to read: %d\n", len);
CHECK(len > 0, "");
do
{
int n = min(len, sizeof(buf));
int nbr = recvfrom(_sock[i], buf, n, 0, NULL, NULL);
printf("n %d nbr %d\n", n, nbr);
CHECK(n == nbr, "");
len -= n;
} while (len);
}
}
return 0;
}
Update 2
Not going to segment messages and using large enough buffer makes the code as follows:
// This is a program which is going to make the bridge between two interfaces. brctl is going to be replaced by this program.
#define Uses_CHECK
#define Uses_close
#define Uses_errno
#define Uses_ETH_P_ALL
#define Uses_FD_SET
#define Uses_htons
#define Uses_ifreq
#define Uses_ioctl
#define Uses_printf
#define Uses_signal
#define Uses_sockaddr_ll
#define Uses_socket
#define Uses_strerror
#include <general.dep>
int _sock[2];
int _CtrlCHandler()
{
printf(" terminating...\n");
CHECK(close(_sock[0]) != -1, "");
CHECK(close(_sock[1]) != -1, "");
printf("all sockets closed successfully.\n");
}
void CtrlCHandler(int dummy)
{
_CtrlCHandler();
}
int OpenSocket(const char *ifname)
{
// getting socket
int socket_fd = socket(PF_PACKET, SOCK_RAW/*|SOCK_NONBLOCK*/, htons(ETH_P_ALL));
CHECK(socket_fd != -1, "%s", strerror(errno));
// init interface options struct with the interface name
struct ifreq if_options;
memset(&if_options, 0, sizeof(struct ifreq));
strncpy(if_options.ifr_name, ifname, sizeof(if_options.ifr_name) - 1);
if_options.ifr_name[sizeof(if_options.ifr_name) - 1] = 0;
// enable promiscuous mode
CHECK(ioctl(socket_fd, SIOCGIFFLAGS, &if_options) != -1, "%s", strerror(errno));
if_options.ifr_flags |= IFF_PROMISC;
CHECK(ioctl(socket_fd, SIOCSIFFLAGS, &if_options) != -1, "%s", strerror(errno));
// get interface index
CHECK(ioctl(socket_fd, SIOCGIFINDEX, &if_options) != -1, "%s", strerror(errno));
// bind socket to the interface
struct sockaddr_ll my_addr;
memset(&my_addr, 0, sizeof(my_addr));
my_addr.sll_family = AF_PACKET;
my_addr.sll_ifindex = if_options.ifr_ifindex;
CHECK(bind(socket_fd, (struct sockaddr *)&my_addr, sizeof(my_addr)) != -1, "%s", strerror(errno));
// socket is ready
return socket_fd;
}
int MakeBridge(const char *if1, const char *if2)
{
_sock[0] = OpenSocket(if1);
CHECK_NO_MSG(_sock[0]);
_sock[1] = OpenSocket(if2);
CHECK_NO_MSG(_sock[1]);
printf("sockets %d and %d opened successfully\n", _sock[0], _sock[1]);
fd_set readfds, orig;
int activity;
char buf[1<<16];
signal(SIGINT, CtrlCHandler);
int packetNumber = 0;
const int numSocks = _countof(_sock);
int nfds = _sock[0];
for (int i = 1; i < numSocks; i++)
if (_sock[i] > nfds)
nfds = _sock[i];
nfds++;
FD_ZERO(&orig);
for (int i = 0; i < numSocks; i++)
FD_SET(_sock[i], &orig);
while (true)
{
readfds = orig;
activity = select(nfds, &readfds, NULL, NULL, NULL);
if (activity == -1) // sockets closed
break;
CHECK(activity > 0, "");
for (int i = 0; i < numSocks; i++)
if (FD_ISSET(_sock[i], &readfds))
{
int len = recvfrom(_sock[i], buf, sizeof(buf), MSG_TRUNC, NULL, NULL);
CHECK(len > 0, "");
CHECK(len <= sizeof(buf), "small size buffer");
printf("%10d %d %d\n", ++packetNumber, i, len);
CHECK(sendto(_sock[!i], buf, len, 0, NULL, 0) == len, "");
}
}
return 0;
}
int Usage(int argc, const char *argv[])
{
printf("Usage: %s <if1> <if2>\n", argv[0]);
printf("Bridges two interfaces with the names specified.\n");
return 0;
}
int main(int argc, const char *argv[])
{
if (argc != 3)
return Usage(argc, argv);
CHECK_NO_MSG(MakeBridge(argv[1], argv[2]));
return 0;
}

RSA signing, same private key but different signature?

nodejs:
const crypto = require(‘crypto’)
const fs = require(‘fs’)
var privateKey = fs.readFileSync(‘private.pem’).toString(‘utf8’)
const sign = crypto.createSign(‘RSA-SHA256’); // RSA-SHA256和SHA256结果一样
sign.update(“test”, ‘acsii’)
var signature = sign.sign(privateKey, ‘hex’)
console.log(signature)
output:
279d0ec2405e29db7024217dd19bc939b0d149a69335f52943ec42f0b40668a38b6d7d0661b71552
d61bf6a541680e4e542fa18e0bcbb860a7f57ec0fb89c3ea8e5cdce2bf6b8c71f0c1462a9cf228e9
eb85b00360201ac0c512a1d701c4bacf2bfc6a256e2e6451a88fb0aad849cf44e0fb947c5e315c52
89ef09eebee5e05
c++:
int main(int argc, char **argv){
ERR_load_ERR_strings();
ERR_load_RSA_strings();
ERR_load_PEM_strings();
ERR_load_BIO_strings();
string strPemFileName="private.pem";
FILE* hPriKeyFile = fopen(strPemFileName.c_str(), "rb");
if( hPriKeyFile == NULL )
{
assert(false);
return 1;
}
RSA* pRSAPriKey = RSA_new();
if(PEM_read_RSAPrivateKey(hPriKeyFile, &pRSAPriKey, 0, 0) == NULL)
{
assert(false);
return 1;
}
unsigned int sig_len = 0;
unsigned int len = 1024;
char * out = (char *)malloc(len);
memset(out, 0, len);
string context="test";
unsigned char md[SHA256_DIGEST_LENGTH] = {0};
if(1 != RSA_sign(NID_sha256,(unsigned char*)context.c_str(), context.size(), (unsigned char*)out, &sig_len,pRSAPriKey))
{
char szErrMsg[LEN_1024] = { 0 };
ERR_error_string_n(ERR_get_error(), szErrMsg, LEN_1024);
cout << szErrMsg << endl;
}
print_hex((unsigned char *)out, sig_len);
free(out);
return 0;
}
output:
3446ed9d60bde475f6f2844c5c98fcedf08578617243ab378231db574b1685f2f82ce50da36263dc
cf6ec9152160dc66c5e0d3a7cfc5de311300cecb29c1b0135b876e024ef13a50d345c4d37bcf53e1
47a235e72cc0a40d160e37eb1abbf4e18deb132e68f5155b324b7ad95daa38c44dd607e0b1d7d1c1
370d6cceebe7ad1e

Can't read received packet when the data section starts with 2

so... context: I'm doing a layer 2 protocol for flexible forwarding in vehicular environment (for now my testbed is in virtual machines), this should take in consideration a different number of interfaces (for multihoming) and multihop.
So what I have:
A way of broadcasting hop-by-hop the service provider.
What I'm triyng to do:
A way to register a session all the way from the client to the provider (And here is the problem)
Problem: I have two types of packets
1st is listened correctly and data payload starts with a 1
2nd for some reason is not detected but I can see the packet is sent and correct with tcpdump
Since I have to register in the application the interface where the connection is made I used select() which seems to be part of the problem since I only guessed how it was used and I'm kind of in the dark about this.
UPDATED v3:
Okay so as soon as I removed most of the stuff about only sending on a specific interface all the stuff worked perfectly (I still need to clean this code... it's kind of messy). Here is code if someone is interested:
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
#include <time.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <string.h>
#include <ifaddrs.h>
#include <signal.h>
#include <unistd.h>
#include <errno.h>
#include <arpa/inet.h>
#include <linux/if_packet.h>
#include <net/ethernet.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <net/if.h>
#include <netinet/in.h>
#define ETH_P_CUSTOM 0x0801 /* EtherType of Current Used Protocol*/
#define BUF_SIZE 1024
typedef enum {
false, true
} Bool; /* Boolean Definition*/
typedef struct Stat {
uint8_t maxSocket; /*Number of sockets to use in receive*/
uint8_t nInterfaces; /*Number of interfaces owned by this machine*/
uint8_t nSession; /*Number of Sessions Known in the linked list*/
uint8_t upMac[ETH_ALEN]; /*MAC of this host upstream parent*/
uint8_t nHops; /*Hops to Provider*/
char ifName[IF_NAMESIZE + 1]; /*Interface to Provider*/
} Stat;
typedef struct Node {
uint64_t session; /*Client Session*/
uint8_t nextHop[ETH_ALEN]; /*Next-Hop to Client*/
char ifName[IF_NAMESIZE + 1]; /*Outgoing Interface that connects to Next-Hop*/
struct Node * next; /*Next Session*/
} Node;
typedef struct ifNode {
uint8_t ifIndex; /*Interface index*/
uint8_t sock; /*Index in array of sockets*/
uint8_t mac[ETH_ALEN]; /*Interface MAC*/
char ifName[IF_NAMESIZE + 1]; /*Interface Name*/
struct ifNode * next; /*Next Session*/
} ifNode;
Stat * op; /* Variable which tracks status of certain structures/variables*/
Node * first = NULL, *last = NULL; /* Edges of linked list */
ifNode * iffirst = NULL, *iflast = NULL; /* Edges of interface linked list */
int cargc;
char **cargv;
int receiveP();
int broadServ();
int announceSelf();
Node* create(uint64_t sess, uint8_t n[ETH_ALEN], char interface[IF_NAMESIZE]);
void insert_node(Node * p);
Node* search(uint64_t session);
void update(uint64_t session, Node * p);
ifNode* createif(uint8_t idx, uint8_t sock, uint8_t ifmac[ETH_ALEN],
char interface[IF_NAMESIZE]);
void insert_ifnode(ifNode * p);
ifNode* searchif(uint8_t idx, uint8_t mode);
void updateif(uint8_t idx, ifNode * p);
void display();
void displayif();
void ctrlcoverride(int sig) {
printf("\nCtrl-C - Signal Caught - Exiting\n\n");
printf(
"Current Upstream MAC: %02x:%02x:%02x:%02x:%02x:%02x - NHops : %u - At Interface %s\n\n",
op->upMac[0], op->upMac[1], op->upMac[2], op->upMac[3],
op->upMac[4], op->upMac[5], op->nHops, op->ifName);
display();
exit(EXIT_SUCCESS);
}
Node* create(uint64_t sess, uint8_t n[ETH_ALEN], char interface[IF_NAMESIZE]) {
Node * new = (Node *) malloc(sizeof(Node));
if (new == NULL) {
printf("Could not create new node\n");
return NULL;
} else {
strcpy(new->ifName, interface);
new->session = sess;
int i;
for (i = 0; i < ETH_ALEN; i++)
new->nextHop[i] = n[i];
new->next = NULL;
return new;
}
}
ifNode* createif(uint8_t idx, uint8_t sock, uint8_t ifmac[ETH_ALEN],
char interface[IF_NAMESIZE]) {
ifNode * new = (ifNode *) malloc(sizeof(ifNode));
if (new == NULL) {
printf("Could not create new interface node\n");
return NULL;
} else {
new->ifIndex = idx;
new->sock = sock;
strcpy(new->ifName, interface);
int i;
for (i = 0; i < ETH_ALEN; i++)
new->mac[i] = ifmac[i];
new->next = NULL;
return new;
}
}
void insert_node(Node * p) {
if (first == last && last == NULL) {
first = last = p;
first->next = NULL;
last->next = NULL;
} else {
last->next = p;
last = last->next;
last->next = NULL;
}
}
void insert_ifnode(ifNode * p) {
if (iffirst == iflast && iflast == NULL) {
iffirst = iflast = p;
iffirst->next = NULL;
iflast->next = NULL;
} else {
iflast->next = p;
iflast = iflast->next;
iflast->next = NULL;
}
}
Node* search(uint64_t session) {
if (first == last && last == NULL) {
return NULL;
} else {
Node * temp;
for (temp = first; temp != NULL; temp = temp->next) {
if (temp->session == session) {
return temp;
}
}
return NULL;
}
}
ifNode* searchif(uint8_t idx, uint8_t mode) {
if (iffirst == iflast && iflast == NULL) {
return NULL;
} else {
ifNode * temp;
for (temp = iffirst; temp != NULL; temp = temp->next) {
if (temp->ifIndex == idx && mode == 0) {
return temp;
} else if (temp->sock == idx && mode == 1) {
return temp;
}
}
return NULL;
}
}
void update(uint64_t session, Node * p) {
if (first == last && last == NULL) {
return;
} else {
Node * temp;
for (temp = first; temp != NULL; temp = temp->next) {
if (temp->session == session) {
strcpy(temp->ifName, p->ifName);
temp->next = p->next;
int i;
for (i = 0; i < ETH_ALEN; i++)
temp->nextHop[i] = p->nextHop[i];
return;
}
}
}
}
void updateif(uint8_t idx, ifNode * p) {
if (iffirst == iflast && iflast == NULL) {
return;
} else {
ifNode * temp;
for (temp = iffirst; temp != NULL; temp = temp->next) {
if (temp->ifIndex == idx) {
strcpy(temp->ifName, p->ifName);
temp->sock = p->sock;
temp->next = p->next;
int i;
for (i = 0; i < ETH_ALEN; i++)
temp->mac[i] = p->mac[i];
return;
}
}
}
}
void display() {
Node * temp = first;
while (temp != NULL) {
printf("Session %" PRIu64 " Through %s - NextHop at ", temp->session,
temp->ifName);
int i;
for (i = 0; i < ETH_ALEN; i++)
printf("%02x ", temp->nextHop[i]);
printf("\n");
temp = temp->next;
}
}
void displayif() {
ifNode * temp = iffirst;
while (temp != NULL) {
printf("Interface Index %u Socket Number %u - Name %s with MAC: ",
temp->ifIndex, temp->sock, temp->ifName);
int i;
for (i = 0; i < ETH_ALEN; i++)
printf("%02x ", temp->mac[i]);
printf("\n");
temp = temp->next;
}
}
uint8_t counter() {
Node * temp = first;
uint8_t counter = 0;
while (temp != NULL) {
counter++;
temp = temp->next;
}
return counter;
}
fd_set rfds;
int rec;
int main(int argc, char **argv) {
setbuf(stdout, NULL);
signal(SIGINT, ctrlcoverride);
cargc = argc;
cargv = argv;
/*Setting Base Variables to Initial Values*/
op = (Stat*) malloc(sizeof(Stat));
op->nSession = 0;
memset(op->ifName, 0, IF_NAMESIZE);
op->maxSocket = 0;
op->nHops = UINT8_MAX - 1;
int i;
for (i = 0; i < ETH_ALEN; i++) {
op->upMac[i] = 0x00;
}
memset(&rfds, 0, sizeof(fd_set));
FD_ZERO(&rfds);
if (argc != 2) {
printf("USAGE: sudo %s {provider|node|nodekey}\n", cargv[0]);
exit(EXIT_FAILURE);
} else if (!(strcmp(cargv[1], "provider") == 0
|| strcmp(cargv[1], "node") == 0 || strcmp(cargv[1], "nodekey") == 0)) {
printf("USAGE: sudo %s {provider|node|nodekey}\n", cargv[0]);
exit(EXIT_FAILURE);
}
if (strcmp(cargv[1], "nodekey") == 0) {
srand(time(NULL));
uint8_t myArray[6] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
insert_node(
create((uint64_t) (100 * ((float) rand() / RAND_MAX)), myArray,
"SOURCE"));
}
struct ifaddrs *ifaddr, *ifa;
if (getifaddrs(&ifaddr) == -1) {
perror("getifaddrs");
exit(EXIT_FAILURE);
}
for (ifa = ifaddr, op->nInterfaces = 0; ifa != NULL; ifa = ifa->ifa_next) {
if (ifa->ifa_addr == NULL)
continue;
if (ifa->ifa_addr->sa_family == AF_PACKET
&& strncmp(ifa->ifa_name, "lo", strlen("lo")) != 0
&& strncmp(ifa->ifa_name, "tap", strlen("tap")) != 0) {
op->nInterfaces++;
}
}
rec = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_CUSTOM));
int sockopt;
char ifName[IFNAMSIZ];
struct ifreq ifr;
for (i = 1, ifa = ifaddr; ifa != NULL;
ifa = ifa->ifa_next, i++) {
if (ifa->ifa_addr == NULL)
continue;
if (ifa->ifa_addr->sa_family == AF_PACKET
&& strncmp(ifa->ifa_name, "lo", strlen("lo")) != 0
&& strncmp(ifa->ifa_name, "tap", strlen("tap")) != 0) {
uint8_t sock;
if ((sock = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_CUSTOM)))
== -1) {
printf("socket() error: %u - %s\n", errno, strerror(errno));
return EXIT_FAILURE;
}
if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &sockopt,
sizeof sockopt) == -1) {
printf("SO_REUSEADDR error: %u - %s\n", errno, strerror(errno));
close(sock);
return EXIT_FAILURE;
}
memset(&ifr, 0, sizeof(struct ifreq));
ifr.ifr_ifindex = i;
strcpy(ifr.ifr_name, ifa->ifa_name);
if (setsockopt(sock, SOL_SOCKET, SO_BINDTODEVICE, ifa->ifa_name,
IF_NAMESIZE) == -1) {
printf("SO_BINDTODEVICE error: %u - %s\n", errno,
strerror(errno));
close(sock);
return EXIT_FAILURE;
}
struct sockaddr_ll sll;
sll.sll_family = AF_PACKET;
sll.sll_ifindex = i;
sll.sll_protocol = htons(ETH_P_CUSTOM);
if ((bind(sock, (struct sockaddr *) &sll, sizeof(sll))) == -1) {
perror("Error binding raw socket to interface\n");
exit(-1);
}
if ((ioctl(sock, SIOCGIFHWADDR, &ifr)) != 0) {
printf("SIOCGIFHWADDR error: %u - %s\n", errno,
strerror(errno));
return EXIT_FAILURE;
}
int j;
uint8_t ifmac[ETH_ALEN];
for (j = 0; j < ETH_ALEN; j++) {
ifmac[j] = (uint8_t) (ifr.ifr_hwaddr.sa_data)[j];
}
FD_SET(sock, &rfds);
op->maxSocket = (op->maxSocket < sock) ? sock : op->maxSocket;
insert_ifnode(createif(i, sock, ifmac, ifr.ifr_name));
}
}
displayif();
if (strcmp(cargv[1], "provider") == 0) {
struct ifreq if_mac; // interface
char * interface = "eth1";
int sockfd;
if ((sockfd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_CUSTOM))) == -1) {
printf("socket() error: %u - %s\n", errno, strerror(errno));
return EXIT_FAILURE;
}
memset(&if_mac, 0, sizeof(struct ifreq));
strncpy(if_mac.ifr_name, interface, IFNAMSIZ - 1);
if ((ioctl(sockfd, SIOCGIFHWADDR, &if_mac)) != 0) {
printf("SIOCGIFHWADDR error: %u - %s\n", errno, strerror(errno));
return EXIT_FAILURE;
}
int i;
for (i = 0; i < ETH_ALEN; i++)
op->upMac[i] = ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[i];
op->nHops = 0;
close(sockfd);
}
freeifaddrs(ifaddr);
int stat = 0;
while (1) {
if (strcmp(cargv[1], "provider") == 0) {
if ((stat = receiveP()) != 0)
return stat;
if ((stat = broadServ()) != 0)
return stat;
display();
usleep(100000);
} else if (strcmp(cargv[1], "node") == 0
|| strcmp(cargv[1], "nodekey") == 0) {
if ((stat = receiveP()) != 0)
return stat;
if ((stat = announceSelf()) != 0){
return stat;
}
if ((stat = broadServ()) != 0)
return stat;
display();
usleep(100000);
}
}
ifNode * temp = iffirst;
while (temp != NULL) {
close(temp->sock);
temp = temp->next;
}
exit(stat);
}
int receiveP() {
int stat = 0;
struct ifreq ifr;
struct sockaddr saddr;
long unsigned int numbytes = 0;
char buf[BUF_SIZE];
memset(buf, 0, BUF_SIZE);
struct ether_header *eh = (struct ether_header *) buf;
unsigned int saddr_size = sizeof saddr;
struct timeval tv;
tv.tv_sec = 3; /* 3 Secs Timeout */
tv.tv_usec = 0;
setsockopt(rec, SOL_SOCKET, SO_RCVTIMEO, (char *) &tv,
sizeof(struct timeval));
numbytes = recvfrom(rec, buf, BUF_SIZE, 0, &saddr, &saddr_size);
int len;
int ntable;
switch (buf[sizeof(struct ether_header)]) {
case 1:
if (buf[sizeof(struct ether_header) + 1] < op->nHops) {
op->upMac[0] = eh->ether_shost[0];
op->upMac[1] = eh->ether_shost[1];
op->upMac[2] = eh->ether_shost[2];
op->upMac[3] = eh->ether_shost[3];
op->upMac[4] = eh->ether_shost[4];
op->upMac[5] = eh->ether_shost[5];
op->nHops = buf[sizeof(struct ether_header) + 1] + 1;
memset(&ifr, 0, sizeof(struct ifreq));
memset(&ifr.ifr_name, 0, IF_NAMESIZE);
printf(
"Server %u Hops Away - Through %02x:%02x:%02x:%02x:%02x:%02x At Interface %s\n",
op->nHops, eh->ether_shost[0], eh->ether_shost[1],
eh->ether_shost[2], eh->ether_shost[3], eh->ether_shost[4],
eh->ether_shost[5], op->ifName);
printf("\n\n");
}
break;
case 2:
len = sizeof(struct ether_header) + 1;
ntable = buf[len++];
int j;
for (j = 0; j < ntable; j++, len++) {
if (search(buf[len]) == NULL) {
insert_node(create(buf[len], eh->ether_shost, ""));
}
}
break;
}
return stat;
}
int broadServ() {
int stat = 0;
int tx_len = 0;
char sendbuf[BUF_SIZE];
char ifName[IF_NAMESIZE - 1];
struct ether_header *eh = (struct ether_header *) sendbuf;
struct sockaddr_ll socket_address;
int i;
struct ifreq ifr, if_mac;
ifNode * temp = iffirst;
while (temp != NULL) {
/* Get the index of the interface to send on */
memset(&ifr, 0, sizeof(struct ifreq));
ifr.ifr_ifindex = temp->ifIndex;
if (ioctl(temp->sock, SIOCGIFNAME, &ifr) < 0)
perror("SIOCGIFINDEX");
memset(ifName, 0, IF_NAMESIZE - 1);
strncpy(ifName, ifr.ifr_name, IF_NAMESIZE - 1);
/* Get the MAC address of the interface to send on */
memset(&if_mac, 0, sizeof(struct ifreq));
strncpy(if_mac.ifr_name, ifName, IFNAMSIZ - 1);
if (ioctl(temp->sock, SIOCGIFHWADDR, &if_mac) < 0)
perror("SIOCGIFHWADDR");
if (((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[0] == 0x00
&& ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[1] == 0x00
&& ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[2] == 0x00
&& ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[3] == 0x00
&& ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[4] == 0x00
&& ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[5] == 0x00)
continue;
memset(sendbuf, 0, BUF_SIZE);
/* Ethernet header */
eh->ether_shost[0] = ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[0];
eh->ether_shost[1] = ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[1];
eh->ether_shost[2] = ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[2];
eh->ether_shost[3] = ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[3];
eh->ether_shost[4] = ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[4];
eh->ether_shost[5] = ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[5];
eh->ether_dhost[0] = 0xff;
eh->ether_dhost[1] = 0xff;
eh->ether_dhost[2] = 0xff;
eh->ether_dhost[3] = 0xff;
eh->ether_dhost[4] = 0xff;
eh->ether_dhost[5] = 0xff;
/* Ethertype field */
eh->ether_type = htons(ETH_P_CUSTOM);
tx_len = sizeof(struct ether_header);
/* Packet data */
sendbuf[tx_len++] = 1;
sendbuf[tx_len++] = op->nHops; //+1;
/* Index of the network device */
socket_address.sll_ifindex = temp->ifIndex;
/* Address length*/
socket_address.sll_halen = ETH_ALEN;
/* Destination MAC */
socket_address.sll_addr[0] = 0xff;
socket_address.sll_addr[1] = 0xff;
socket_address.sll_addr[2] = 0xff;
socket_address.sll_addr[3] = 0xff;
socket_address.sll_addr[4] = 0xff;
socket_address.sll_addr[5] = 0xff;
/* Send packet */
if (sendto(temp->sock, sendbuf, tx_len, 0,
(struct sockaddr*) &socket_address, sizeof(struct sockaddr_ll))
< 0)
printf("Send failed\n");
temp = temp->next;
}
return stat;
}
int announceSelf() {
if (op->upMac[0] == 0x00 && op->upMac[1] == 0x00 && op->upMac[2] == 0x00
&& op->upMac[3] == 0x00 && op->upMac[4] == 0x00
&& op->upMac[5] == 0x00)
return EXIT_SUCCESS;
int stat = 0;
int tx_len = 0;
char sendbuf[BUF_SIZE];
char ifName[IF_NAMESIZE - 1];
struct ether_header *eh = (struct ether_header *) sendbuf;
struct sockaddr_ll socket_address;
int i;
struct ifreq ifr, if_mac;
ifNode * temp = iffirst;
while (temp != NULL) {
memset(&ifr, 0, sizeof(struct ifreq));
ifr.ifr_ifindex = temp->ifIndex;
if (ioctl(temp->sock, SIOCGIFNAME, &ifr) < 0)
perror("SIOCGIFINDEX");
memset(ifName, 0, IF_NAMESIZE - 1);
strncpy(ifName, ifr.ifr_name, IF_NAMESIZE - 1);
/* Get the MAC address of the interface to send on */
memset(&if_mac, 0, sizeof(struct ifreq));
strncpy(if_mac.ifr_name, ifName, IFNAMSIZ - 1);
if (ioctl(temp->sock, SIOCGIFHWADDR, &if_mac) < 0)
perror("SIOCGIFHWADDR");
if (((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[0] == 0x00
&& ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[1] == 0x00
&& ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[2] == 0x00
&& ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[3] == 0x00
&& ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[4] == 0x00
&& ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[5] == 0x00)
continue;
memset(sendbuf, 0, BUF_SIZE);
/* Ethernet header */
eh->ether_shost[0] = ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[0];
eh->ether_shost[1] = ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[1];
eh->ether_shost[2] = ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[2];
eh->ether_shost[3] = ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[3];
eh->ether_shost[4] = ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[4];
eh->ether_shost[5] = ((uint8_t *) &if_mac.ifr_hwaddr.sa_data)[5];
eh->ether_dhost[0] = op->upMac[0];
eh->ether_dhost[1] = op->upMac[1];
eh->ether_dhost[2] = op->upMac[2];
eh->ether_dhost[3] = op->upMac[3];
eh->ether_dhost[4] = op->upMac[4];
eh->ether_dhost[5] = op->upMac[5];
/* Ethertype field */
eh->ether_type = htons(ETH_P_CUSTOM);
tx_len = sizeof(struct ether_header);
/* Packet data */
sendbuf[tx_len++] = 2;
sendbuf[tx_len++] = counter();
Node *temp1 = first;
for (; temp1 != NULL; temp1 = temp1->next) {
sendbuf[tx_len++] = temp1->session;
}
socket_address.sll_ifindex = temp->ifIndex;
/* Address length*/
socket_address.sll_halen = ETH_ALEN;
/* Destination MAC */
socket_address.sll_addr[0] = op->upMac[0];
socket_address.sll_addr[1] = op->upMac[1];
socket_address.sll_addr[2] = op->upMac[2];
socket_address.sll_addr[3] = op->upMac[3];
socket_address.sll_addr[4] = op->upMac[4];
socket_address.sll_addr[5] = op->upMac[5];
/* Send packet */
if (sendto(temp->sock, sendbuf, tx_len, 0,
(struct sockaddr*) &socket_address, sizeof(struct sockaddr_ll))
< 0)
printf("Send failed\n");
temp = temp->next;
}
return stat;
}
So to test this you can have VM with linux connected like this (for example):
Provider ----- Node ----- Node ----- Nodekey
I still had a problem when creating multiple sessions, i wasn't incrementing the buffer when reading and I was reading multiple times the same position. Now it's working good
OK, let's begin with the easiest recommendations but I'm not sure this is going to resolve the problem at once. I did a system like this many years ago for different boards with different processor architectures communicating with each other. All the boards were running within a telecommunication switch. It's a very nice problem and you are facing it in the proper way with a peer-to-peer distributed solution.
I didn't go through all code but it seems each node is discovering the neighbour nodes in the network and everyone is creating a tree.
In select, the first argument should not be FD_SETSIZE but the highest-numbered file descriptor in any of the three sets (in this case the read set), plus 1.
The infinite loop is calling receiveSession which is creating all sockets again and then it reads. If a frame with your specific layer-2 protocol arrives in the middle and there is no socket listening for it, it will be discarded. Maybe your problem could be here.
When you send Ethernet frames directly, the hardware will complete the frame to the minimum Ethernet size: 64 octets (so you might receive padding data up to 46 octets - Octets not Bytes)
Please read here:
http://en.wikipedia.org/wiki/Ethernet_frame
It is good you chose an EtherType ETH_P_CUSTOM higher than 1536 that is not already in use but maybe you want to use a much higher number in order to minimize possibilities of collision with other protocols.
Something important. Your testbed now is with VM's which are usually x86 architectures, 64 bits. When you run your software in real devices with different processors, that might not be the situation. This is very important because you might have different architectures with different endianship and different integer size. That will affect the integer numbers you send, especially in ether_header, and the size of your structures. You have to use the the macros ntohs, ntohl, htons, htonl to change between host and network endianship (session is uint64_t). You should send data in network endianship. This is not solving your very current problem but you might have this problem in the future.

Getting MAC address issue Linux(Ubuntu)

I have got MAC address as under.
struct ifreq ifr;
struct ifreq *IFR;
struct ifconf ifc;
char buf[1024];
int s, i;
int ok = 0;
string macAddr = "";
s = socket(AF_INET, SOCK_DGRAM, 0);
if (s==-1) {
return;
}
ifc.ifc_len = sizeof(buf);
ifc.ifc_buf = buf;
ioctl(s, SIOCGIFCONF, &ifc);
IFR = ifc.ifc_req;
for (i = ifc.ifc_len / sizeof(struct ifreq); --i >= 0; IFR++) {
strcpy(ifr.ifr_name, IFR->ifr_name);
if (ioctl(s, SIOCGIFFLAGS, &ifr) == 0) {
if (! (ifr.ifr_flags & IFF_LOOPBACK)) {
if (ioctl(s, SIOCGIFHWADDR, &ifr) == 0) {
ok = 1;
break;
}
}
}
}
close(s);
int p = sizeof(ifr.ifr_hwaddr.sa_data);
cout<<"\n Size:"<<p<<"\n";
for(int i = 0; i < p; i++)
macAddr += ifr.ifr_hwaddr.sa_data[i];
cout<<"\n MAC Address:"<<macAddr<<"\n";
I got ifr.ifr_hwaddr.sa_data data proper, but when I print out I am not getting proper value. What can be the issue for it.
The address in sa_data isn't in text form, it's the raw binary form, so you need to format it as hex yourself. You also can't use sizeof to get the size of the address as sa_data is a generically sized array - you need to look at the sa_len member to get the length of this address. In C you want something like this:
char *separator = "";
for (int i = 0; i < ifr.ifr_hwaddr.sa_len; i++)
{
printf("%s%02x", separator, ifr.ifr_hwaddr.sa_data[i]);
separator = ":";
}
printf("\n");

Resources