/* Copyright Coraid, Inc. 2010. All Rights Reserved. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define nelem(x) (sizeof(x)/sizeof((x)[0])) #define nil NULL #define vprintf(...) if (qflag) ; else fprintf(stderr, __VA_ARGS__) #define mintu(x) ((x) > 60 ? x : 60) typedef unsigned char uchar; typedef unsigned long ulong; typedef struct Aoe Aoe; typedef struct Qc Qc; typedef struct Ata Ata; typedef struct Lun Lun; typedef struct Eth Eth; typedef struct Targ Targ; typedef struct Mac Mac; struct Aoe { uchar dst[6]; uchar src[6]; uchar type[2]; uchar flags; uchar error; uchar major[2]; uchar minor; uchar cmd; uchar tag[4]; }; struct Qc { Aoe h; uchar bufcnt[2]; uchar firmware[2]; uchar scnt; uchar vercmd; uchar len[2]; // uchar data[1024]; }; struct Ata { Aoe h; uchar aflag; uchar err; uchar scnt; uchar cmd; uchar lba[6]; uchar res[2]; }; struct Lun { Lun *next; int state; char ea[6]; int major; int minor; int nsect; int maxsect; int id; }; struct Eth { Lun *luns; int fd; char *name; char ea[6]; int mtu; int up; uchar pkt[16*1024]; }; struct Targ { Targ *next; int major; int minor; }; struct Mac { Mac *next; char ea[6]; }; enum { Neth= 16, Nstack= 16*1024, Nws= 5, Lnew= 0, Lprobe, Arsp= (1<<3), Aerr= (1<<2), Cata= 0, Cqc= 1, ETaoe= 0x88a2, }; int ethlist(char **, int); int ethopen(Eth *); void *jcheck(void *); int jinput(Eth *); Lun *findlun(Eth *e, Aoe *a); void jprobe(Eth *e, Lun *lun); void printlist(void); void printsancheck(void); ulong nhgetl(uchar *); ushort nhgets(uchar *); void hnputs(uchar *, ushort); void hnputl(uchar *, ulong); void *mallocz(int); void inserttarg(int, int); void insertmac(Mac **, char *); void sancheck(int, int); void ifsummary(void); int ifup(char *); void inserteth(char **, int, char *); char *getpciid(char *, int, char *); Eth eth[Neth]; int waitsecs = Nws; pthread_t threads[Neth]; Targ *targlist; int main(int argc, char **argv) { int n, i; char *ethnames[Neth]; Eth *e; memset(ethnames, 0, sizeof ethnames); printf("Probing..."); fflush(0); n = ethlist(ethnames, nelem(ethnames)); for (i=0; iname = ethnames[i]; e->up = ifup(ethnames[i]); if (pthread_create(&threads[i], 0, jcheck, e)) { fprintf(stderr, "pthread_create failed.\n"); break; } } n = i; for (i=0; i> 4) & 0xf)]; *p++ = hex[ea[i] & 0xf]; } *p = 0; return op; } void printlist(void) { Eth *e; Lun *lun; char mac[13]; for (e=eth; e->name; e++) { printf("%s:\n", e->name); for (lun=e->luns; lun; lun=lun->next) printf("e%d.%d %s %d\n", lun->major, lun->minor, cea(mac, lun->ea), lun->maxsect); printf("\n"); } } void timewait(int secs) /* arrange for a sig_alarm signal after `secs' seconds */ { struct sigaction sa; void catch(int); memset(&sa, 0, sizeof sa); sa.sa_handler = catch; sa.sa_flags = SA_RESETHAND; sigaction(SIGALRM, &sa, NULL); alarm(secs); } int discover(Eth *e) { Aoe *a; Qc *q; memset(e->pkt, 0, sizeof e->pkt); a = (Aoe *) e->pkt; memset(a->dst, 0xff, 6); memmove(a->src, e->ea, 6); hnputs(a->type, ETaoe); hnputl(a->tag, 1<<31); hnputs(a->major, 0xffff); a->minor = 0xff; a->cmd = Cqc; a->flags = 0x10; if (write(e->fd, a, mintu(sizeof *q)) <= 0) return -1; return 0; } void * jcheck(void *v) { Eth *e = v; int n; time_t t, nt; struct pollfd pd; if (ethopen(e) < 0) return 0; if (discover(e) < 0) { fprintf(stderr, "skipping %s, discover failure: %s\n", e->name, strerror(errno)); return 0; } pd.fd = e->fd; pd.events = POLLIN; t = time(0); for (;;) { nt = time(0); if (nt-t >= waitsecs) return 0; if (poll(&pd, 1, waitsecs*1000) > 0) if ((n = read(e->fd, e->pkt, sizeof e->pkt)) > 0) if (jinput(e)) t = time(0); } } /* return 1 == useful, 0 == not useful */ int jinput(Eth *e) { Aoe *a; Qc *q; Ata *aa; Lun *lun; ulong tag; int n; a = (Aoe *) e->pkt; if ((a->flags & (Arsp|Aerr)) != Arsp) return 0; if ((a->tag[0] & 0x80) == 0) return 0; tag = nhgetl(a->tag); switch (a->cmd) { case Cqc: q = (Qc *) a; lun = findlun(e, a); if (lun->state == Lnew) { lun->nsect = q->scnt; jprobe(e, lun); lun->state = Lprobe; break; } return 0; case Cata: aa = (Ata *) a; lun = findlun(e, a); if (lun == nil) return 0; if (lun->id != tag>>16) { printf("lun->id %d != tag %ld for %d.%d\n", lun->id, tag>>16, lun->major, lun->minor); return 0; } n = tag & 0xff; if (n > lun->maxsect) lun->maxsect = n; break; default: return 0; } return 1; } void hnputl(uchar *p, ulong n) { *p++ = n >> 24; *p++ = n >> 16; *p++ = n >> 8; *p = n; } void hnputs(uchar *p, ushort s) { *p++ = s >> 8; *p = s; } ushort nhgets(uchar *p) { ushort s; s = *p++; s <<= 8; s += *p++ & 0xff; return s; } ulong nhgetl(uchar *p) { ulong n; n = *p++; n <<= 8; n += *p++ & 0xff; n <<= 8; n += *p++ & 0xff; n <<= 8; n += *p++ & 0xff; return n; } void jprobe(Eth *e, Lun *lun) { Aoe *a; Ata *aa; int n; memset(e->pkt, 0, sizeof e->pkt); a = (Aoe *) e->pkt; aa = (Ata *) a; memcpy(a->dst, lun->ea, 6); memcpy(a->src, e->ea, 6); hnputs(a->type, ETaoe); hnputs(a->major, lun->major); a->minor = lun->minor; a->flags = 0x10; hnputl(a->tag, lun->id<<16); aa->cmd = 0xec; aa->scnt = 1; n = e->mtu - sizeof *aa; for (n &= ~511; n > 0; n -= 512) { a->tag[3] = n/512; if (write(e->fd, a, sizeof *aa + n) <= 0) { printf("write failed\n"); } usleep(100); } } Lun * findlun(Eth *e, Aoe *a) { Lun *p, **pp; int maj, n; static int id; maj = nhgets(a->major); pp = &e->luns; for (; (p=*pp); pp=&p->next) { if (maj < p->major) continue; if (maj > p->major) break; if (a->minor < p->minor) continue; if (a->minor > p->minor) break; n = memcmp(p->ea, a->src, 6); if (n < 0) continue; if (n > 0) break; return p; } if (a->cmd == Cata) return nil; p = mallocz(sizeof *p); p->major = maj; p->minor = a->minor; memmove(p->ea, a->src, 6); p->next = *pp; p->id = 0x8000 | id++; inserttarg(p->major, p->minor); return *pp = p; } void catch(int sig) { } int getindx(int sfd, char *name) // return the index of device 'name' { struct ifreq xx; int n; strcpy(xx.ifr_name, name); n = ioctl(sfd, SIOCGIFINDEX, &xx); if (n == -1) return -1; return xx.ifr_ifindex; } int getmtu(Eth *e) { struct ifreq xx; int n; strcpy(xx.ifr_name, e->name); n = ioctl(e->fd, SIOCGIFMTU, &xx); if (n == -1) { perror("Can't get mtu"); return 1500; } return xx.ifr_mtu; } int ethopen(Eth *e) // get us a raw connection to an interface { int n, sfd, rbsz, sbsz; struct sockaddr_ll sa; struct ifreq xx; rbsz = 64*1024*1024; sbsz = 64*1024*1024; memset(&sa, 0, sizeof sa); memset(&xx, 0, sizeof xx); sfd = socket(PF_PACKET, SOCK_RAW, htons(ETaoe)); if (sfd == -1) { perror("got bad socket"); return -1; } if (setsockopt(sfd, SOL_SOCKET, SO_RCVBUFFORCE, &rbsz, sizeof rbsz) < 0) fprintf(stderr, "Failed to set socket rcvbuf size\n"); if (setsockopt(sfd, SOL_SOCKET, SO_SNDBUFFORCE, &sbsz, sizeof sbsz) < 0) fprintf(stderr, "Failed to set socket sndbuf size\n"); n = getindx(sfd, e->name); sa.sll_family = AF_PACKET; sa.sll_protocol = htons(ETaoe); sa.sll_ifindex = n; n = bind(sfd, (struct sockaddr *)&sa, sizeof sa); if (n == -1) { perror("bind funky"); return -1; } strcpy(xx.ifr_name, e->name); n = ioctl(sfd, SIOCGIFHWADDR, &xx); if (n == -1) { perror("Can't get hw addr"); return -1; } memmove(e->ea, xx.ifr_hwaddr.sa_data, 6); e->fd = sfd; e->mtu = getmtu(e); return 0; } int ethlist(char **ifs, int nifs) { int i, s, n; struct ifreq ifr; s = socket(AF_INET, SOCK_STREAM, 0); if (s < 0) return 0; n = 0; for (i=0; imajor = maj; nt->minor = min; if (targlist == NULL) { targlist = nt; return; } if (nt->major < targlist->major) { nt->next = targlist; targlist = nt; return; } else if (nt->major == targlist->major && nt->minor < targlist->minor) { nt->next = targlist; targlist = nt; return; } else if (nt->major == targlist->major && nt->minor == targlist->minor) return; for (p = targlist,t = targlist->next; t; p = t,t = t->next) { if (nt->major == t->major && nt->minor == t->minor) return; if (nt->major < t->major) { p->next = nt; nt->next = t; return; } else if (nt->major == t->major && nt->minor < t->minor) { p->next = nt; nt->next = t; return; } } p->next = nt; } void printsancheck() { Targ *t; printf("==========================================\n"); printf("INTERFACE SUMMARY\n"); printf("==========================================\n"); printf("Name\tStatus\tMTU\tPCI ID\n"); ifsummary(); printf("==========================================\n"); printf("DEVICE SUMMARY\n"); printf("==========================================\n"); printf("Device\tMacs\tPayload\tLocal Interfaces\n"); for (t = targlist; t; t = t->next) { sancheck(t->major, t->minor); } } void ifsummary() { Eth *e; char buf[32]; char *p; for (e=eth; e->name; e++) { p = getpciid(buf, sizeof buf, e->name); printf("%s\t%s\t%d\t%s\n", e->name, (e->up ? "UP" : "DN"), e->mtu, (p ? p : "")); } } void sancheck(int maj, int min) { Eth *e; Lun *l; Mac *ml, *m; int a, found; int ps, nsect, nea, nloc; int mtu, mtuflag; char buf[128]; char mac[13]; nsect = mtu = 0; mtuflag = 0; nloc = nea = 0; a = 0; ps = 0; memset(buf, 0, sizeof buf); ml = NULL; for (e=eth; e->name; e++) { found = 0; for (l = e->luns; l; l = l->next) { if (!(l->major == maj && l->minor == min)) continue; found = 1; if (mtu == 0) mtu = e->mtu; else if (mtu != e->mtu) { mtuflag = 1; mtu = (e->mtu > mtu ? e->mtu : mtu); } insertmac(&ml, l->ea); if (ps == 0) ps = l->maxsect; else ps = (l->maxsect < ps ? l->maxsect : ps); nsect = l->nsect; } if (found) { snprintf(buf + a,(sizeof buf) - a, "%s%s", (a ? ",": ""), e->name); a += strlen(e->name) + (a ? 1 : 0); nloc++; } } for (m = ml; m; ml = m, nea++) { m = m->next; free(ml); } printf("e%d.%d\t%4d\t%d\t%s\n", maj, min, nea, nsect*512, buf); if (nea != nloc) printf(" Mismatched number of local vs remote interfaces\n"); for (e=eth; e->name; e++) { found = 0; for (l = e->luns; l; l = l->next) if (l->major == maj && l->minor == min) { found = 1; if (l->maxsect < (e->mtu-32)/512) printf(" The path %s->%s is only capable of %d byte payloads\n", e->name, cea(mac, l->ea), l->maxsect*512); } if (found) { if (e->mtu < nsect*512 + 36) printf(" %s: MTU (%d) not set optimally for device's capable payload\n",e->name, e->mtu); if (e->mtu < mtu) printf(" %s: MTU different from other interfaces (%d vs. %d)\n", e->name, e->mtu, mtu); } } } void insertmac(Mac **ml, char *m) { Mac *nm,*p, *pp; for (p = *ml, pp=NULL; p; pp = p, p = p->next) if (memcmp(p->ea, m, 6) == 0) return; nm = mallocz(sizeof *nm); memcpy(nm->ea, m, 6); if (pp) pp->next = nm; else *ml = nm; } int ifup(char *ethname) { struct ifreq ifr; int r, s; memset(&ifr, 0, sizeof ifr); s = socket(AF_INET, SOCK_STREAM, 0); if (s < 0) return 0; strcpy(ifr.ifr_name, ethname); if (ioctl(s, SIOCGIFFLAGS, &ifr) < 0) { close(s); return 0; } r = ifr.ifr_flags & IFF_UP; close(s); return r; } void inserteth(char **ifs, int nifs, char *n) { int i, j; char a[64], b[64]; memset(a, 0, sizeof a); memset(b, 0, sizeof b); for (i=0; i < nifs; i++) { if (ifs[i] == 0) { ifs[i] = strdup(n); break; } j = strcmp(n, ifs[i]); if (j < 0) { strcpy(a, n); for (; ifs[i]; i++) { strcpy(b, ifs[i]); free(ifs[i]); ifs[i] = strdup(a); strcpy(a, b); } ifs[i] = strdup(a); break; } else if (j == 0) break; else if (j > 0) continue; } } char * getpciid(char *b, int blen, char *n) { FILE *fd; char dev[8]; char ven[8]; char path[128]; memset(dev, 0, sizeof dev); memset(ven, 0, sizeof ven); memset(path, 0, sizeof path); sprintf(path, "/sys/class/net/%s/device/vendor", n); if ((fd = fopen(path, "r")) == NULL) return NULL; fseek(fd, 2, SEEK_SET); if (fread(ven, 1, 4, fd) <= 0) { fclose(fd); return NULL; } fclose(fd); sprintf(path, "/sys/class/net/%s/device/device", n); if ((fd = fopen(path, "r")) == NULL) return NULL; fseek(fd, 2, SEEK_SET); if (fread(dev, 1, 4, fd) <= 0) { fclose(fd); return NULL; } fclose(fd); snprintf(b, blen, "%s:%s", ven, dev); return b; }