wget-ftpparse patch 0.52, alpha. 19990705 Copyright 1999 Tetsu Ushijima, tu@pobox.com http://pobox.com/~tu/wget-ftpparse.html This is an unofficial patch for GNU Wget, a free network utility to retrieve files from the World Wide Web using HTTP and FTP. It modifies Wget so that it will use D. J. Bernstein's ftpparse library for parsing LIST responses from FTP servers. This means that this modified Wget can understand various LIST output formats, while the original Wget can recognize UNIX-ls-style listings only. This patch is intended to work with Wget version 1.5.3 and ftpparse version 19970712. See http://pobox.com/~djb/ftpparse.html for more information about the ftpparse package. How to use this patch: The following instructions assumes that the relevant four files, namely wget-1.5.3.tar.gz, ftpparse.c, ftpparse.h, and this file, are placed in the current working directory. 1. Unpack the Wget package: % gzip -dc wget-1.5.3.tar.gz | tar xf - 2. Go to the top source directory: % cd wget-1.5.3 3. Apply this patch: % patch -p1 < ../wget-1.5.3-ftpparse-19970712-0.52.patch 4. Copy the ftpparse package to the src directory: % cp ../ftpparse.[ch] src 5. Go ahead with the original Wget installation instructions; see the file INSTALL. That's it! Index: wget/src/Makefile.in diff -c wget/src/Makefile.in:1.1.1.1 wget/src/Makefile.in:1.2 *** wget/src/Makefile.in:1.1.1.1 Thu Oct 8 21:35:47 1998 --- wget/src/Makefile.in Mon Oct 19 23:37:21 1998 *************** *** 59,65 **** OBJ = $(ALLOCA) cmpt$o connect$o fnmatch$o ftp$o ftp-basic$o \ ftp-ls$o $(OPIE_OBJ) getopt$o headers$o host$o html$o \ http$o init$o log$o main$o $(MD5_OBJ) netrc$o rbuf$o \ ! recur$o retr$o url$o utils$o version$o .SUFFIXES: .SUFFIXES: .c .o ._c ._o --- 59,65 ---- OBJ = $(ALLOCA) cmpt$o connect$o fnmatch$o ftp$o ftp-basic$o \ ftp-ls$o $(OPIE_OBJ) getopt$o headers$o host$o html$o \ http$o init$o log$o main$o $(MD5_OBJ) netrc$o rbuf$o \ ! recur$o retr$o url$o utils$o version$o ftpparse$o .SUFFIXES: .SUFFIXES: .c .o ._c ._o *************** *** 137,145 **** connect$o: config.h wget.h sysdep.h options.h connect.h host.h fnmatch$o: config.h wget.h sysdep.h options.h fnmatch.h ftp-basic$o: config.h wget.h sysdep.h options.h utils.h rbuf.h connect.h host.h ! ftp-ls$o: config.h wget.h sysdep.h options.h utils.h ftp.h rbuf.h ftp-opie$o: config.h wget.h sysdep.h options.h md5.h ftp$o: config.h wget.h sysdep.h options.h utils.h url.h rbuf.h retr.h ftp.h html.h connect.h host.h fnmatch.h netrc.h getopt$o: wget.h sysdep.h options.h headers$o: config.h wget.h sysdep.h options.h connect.h rbuf.h headers.h host$o: config.h wget.h sysdep.h options.h utils.h host.h url.h --- 137,146 ---- connect$o: config.h wget.h sysdep.h options.h connect.h host.h fnmatch$o: config.h wget.h sysdep.h options.h fnmatch.h ftp-basic$o: config.h wget.h sysdep.h options.h utils.h rbuf.h connect.h host.h ! ftp-ls$o: config.h wget.h sysdep.h options.h utils.h ftp.h rbuf.h ftpparse.h ftp-opie$o: config.h wget.h sysdep.h options.h md5.h ftp$o: config.h wget.h sysdep.h options.h utils.h url.h rbuf.h retr.h ftp.h html.h connect.h host.h fnmatch.h netrc.h + ftpparse$o: ftpparse.h getopt$o: wget.h sysdep.h options.h headers$o: config.h wget.h sysdep.h options.h connect.h rbuf.h headers.h host$o: config.h wget.h sysdep.h options.h utils.h host.h url.h Index: wget/src/ftp-ls.c diff -c wget/src/ftp-ls.c:1.1.1.1 wget/src/ftp-ls.c:1.4 *** wget/src/ftp-ls.c:1.1.1.1 Thu Oct 8 21:35:47 1998 --- wget/src/ftp-ls.c Fri Dec 4 00:03:47 1998 *************** *** 36,41 **** --- 36,42 ---- #include "wget.h" #include "utils.h" #include "ftp.h" + #include "ftpparse.h" /* Converts symbolic permissions to number-style ones, e.g. string rwxr-xr-x to 755. For now, it knows nothing of *************** *** 373,378 **** --- 374,582 ---- return dir; } + + static int + scan_perms (const char *s, int *perms) + { + int p; + int i; + + if ((*s != '-') && (*s != 'd') && (*s != 'l')) + return 0; + + p = 0; + + for (i = 0; i < 9; ++i) + { + ++s; + p <<= 1; + if (*s == '-') + continue; + switch (i % 3) + { + case 0: + if (*s != 'r') + return 0; + break; + case 1: + if (*s != 'w') + return 0; + break; + case 2: + if ((*s != 'x') && (*s != 's')) + return 0; + break; + } + p |= 1; + } + + *perms = p; + return 1; + } + + /* Convert the directory listing stored in FILE to a linked list of + fileinfo (system-independent) entries. */ + static struct fileinfo * + ftp_parse_generic_ls (const char *file) + { + static struct ftpparse fp; + FILE *listing; + struct fileinfo *dir; + struct fileinfo *last; + char *line; + int len; + struct fileinfo cur; + struct fileinfo *new; + int error; + int ignore; + + listing = fopen (file, "rb"); + if (!listing) + { + logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno)); + return NULL; + } + + dir = NULL; + last = NULL; + + /* Line loop to end of file: */ + while ((line = read_whole_line (listing))) + { + DEBUGP (("%s\n", line)); + len = strlen (line); + /* Destroy if there is one. */ + if (len && line[len - 1] == '\r') + line[--len] = '\0'; + + /* Reset current fileinfo. */ + cur.type = FT_UNKNOWN; + cur.name = NULL; + cur.size = 0; + cur.tstamp = 0; + cur.perms = 0; + cur.linkto = NULL; + + if (!ftpparse (&fp, line, len)) + { + free (line); + continue; + } + + cur.name = (char *)xmalloc (fp.namelen + 1); + memcpy (cur.name, fp.name, fp.namelen); + cur.name[fp.namelen] = '\0'; + + cur.size = fp.size; + cur.tstamp = fp.mtime; + + if (fp.flagtrycwd) + { + cur.type = FT_DIRECTORY; + cur.perms = 0755; + } + + if (fp.flagtryretr) + { + cur.type = FT_PLAINFILE; + cur.perms = 0644; + } + + if (scan_perms (line, &cur.perms)) + if (line[0] == 'l') + { + int i; + + cur.type = FT_SYMLINK; + i = fp.name - line + fp.namelen; + if (line[i] == ' ') + if (line[i + 1] == '-') + if (line[i + 2] == '>') + if (line[i + 3] == ' ') + cur.linkto = xstrdup (line + i + 4); + } + + switch (cur.type) + { + case FT_PLAINFILE: + DEBUGP (("PLAINFILE; ")); + break; + case FT_DIRECTORY: + DEBUGP (("DIRECTORY; ")); + break; + case FT_SYMLINK: + DEBUGP (("SYMLINK; ")); + break; + case FT_UNKNOWN: + DEBUGP (("UNKNOWN; ")); + break; + } + DEBUGP (("perms %03o; ", cur.perms)); + if (cur.linkto) + { + DEBUGP (("link to: %s\n", cur.linkto)); + } + + error = ignore = 0; /* Errnoeous and ignoring entries are + treated equally for now. */ + + /* "." and ".." are an exception! */ + if (!strcmp (cur.name, ".") || !strcmp (cur.name, "..")) + { + DEBUGP (("\nIgnoring `.' and `..'; ")); + ignore = 1; + } + + if ((cur.type == FT_DIRECTORY) && (cur.name[fp.namelen - 1] == '/')) + { + cur.name[fp.namelen - 1] = '\0'; + DEBUGP (("trailing `/' on dir.\n")); + } + else if ((cur.type == FT_SYMLINK) && (cur.name[fp.namelen - 1] == '@')) + { + cur.name[fp.namelen - 1] = '\0'; + DEBUGP (("trailing `@' on link.\n")); + } + else if ((cur.type == FT_PLAINFILE) + && (cur.perms & 0111) + && (cur.name[fp.namelen - 1] == '*')) + { + cur.name[fp.namelen - 1] = '\0'; + DEBUGP (("trailing `*' on exec.\n")); + } + + if ((!cur.name) || ((cur.type == FT_SYMLINK) && (!cur.linkto))) + error = 1; + + DEBUGP (("\n")); + + if (error || ignore) + { + DEBUGP (("Skipping.\n")); + FREE_MAYBE (cur.name); + FREE_MAYBE (cur.linkto); + free (line); + continue; + } + + /* Append a new fileinfo to dir. */ + new = (struct fileinfo *)xmalloc (sizeof (struct fileinfo)); + memcpy (new, &cur, sizeof (cur)); + new->next = NULL; + new->prev = last; + if (last) + last->next = new; + else + dir = new; + last = new; + + free (line); + } + + fclose (listing); + return dir; + } + /* This function is just a stub. It should actually accept some kind of information what system it is running on -- e.g. FPL_UNIX, FPL_DOS, FPL_NT, FPL_VMS, etc. and a "guess-me" value, like *************** *** 384,388 **** --- 588,597 ---- struct fileinfo * ftp_parse_ls (const char *file) { + struct fileinfo *dir; + + dir = ftp_parse_generic_ls (file); + if (dir) + return dir; return ftp_parse_unix_ls (file); }