finddupes.c

/* finddupes.c */

/* Search for (and optionally delete) duplicate files in multiple	*/
/* directories based on size and either MD5 or CRC-32 "chksums".	*/

#include <stdio.h>
#include <time.h>

#include "dirwrap.h"
#ifdef USE_MD5
	#include "md5.h"
#else	/* CRC-32 */
	#include "crc32.h"
#endif

typedef struct {
	char	path[MAX_PATH+1];
	long	length;
	time_t	date;
#ifdef USE_MD5
	BYTE	chksum[MD5_DIGEST_SIZE];
#else
	ulong	chksum;
#endif
} file_t;

file_t* file;
ulong	file_count=0;

int fchksum(const char* fname, long length,
#ifdef USE_MD5
			BYTE*
#else
			ulong*
#endif
			chksum)
{
	BYTE* buf=NULL;
	FILE* fp;

	if((fp=fopen(fname,"rb"))==NULL) {
		perror(fname);
		return(-1);
	}
	
	if(length && (buf=malloc(length))==NULL) {
		printf("!Error allocating %ld bytes of memory for %s\n"
			,length,fname);
		fclose(fp);
		return(-1);
	}

	if(fread(buf,sizeof(BYTE),length,fp) != length) {
		perror(fname);
		fclose(fp);
		FREE_AND_NULL(buf);
		return(-1);
	}

	fclose(fp);
#ifdef USE_MD5
	MD5_calc(chksum, buf, length);
#else
	*chksum = crc32(buf, length);
#endif
	FREE_AND_NULL(buf);
	return(0);
}

char* timestr(void)
{
	char* p;
	time_t t=time(NULL);
	p=ctime(&t);
	p[19]=0;		/* chop off year and \n */
	return(p+4);	/* skip day-of-week */
}
	
int searchdir(const char* path, BOOL recursive, ulong compare_bytes)
{
	DIR* dir;
	struct dirent* ent;
	file_t* fp;
	char fpath[MAX_PATH+1];

	printf("%s begin searching %s\n",timestr(), path);
	if((dir = opendir(path))==NULL) {
		perror(path);
		return(1);
	}

	while((ent = readdir(dir))!=NULL) {
		if(kbhit())
			break;
		if(strcmp(ent->d_name,".")==0 || strcmp(ent->d_name,"..")==0)
			continue;
		strcpy(fpath,path);
		backslash(fpath);
		strcat(fpath,ent->d_name);
		if(isdir(fpath)) {
			if(recursive)
				searchdir(fpath, recursive, compare_bytes);
			continue;
		}

		file=realloc(file,sizeof(file_t)*(file_count+1));
		if(file==NULL) {
			printf("!Error allocating %lu bytes\n",sizeof(file_t)*(file_count+1));
			exit(1);
		}
		fp=&file[file_count];
		memset(fp,0,sizeof(file_t));
		strcpy(fp->path,fpath);
		fp->date=fdate(fp->path);
		if((fp->length=flength(fp->path))==-1) {
			printf("!Failed to get length of %s\n",fp->path);
			continue;
		}
		if(compare_bytes && fp->length > compare_bytes)
			fp->length = compare_bytes;
		if(fchksum(fp->path, fp->length,
#ifdef USE_MD5
			fp->chksum
#else
			&fp->chksum
#endif
			))
			continue;
		file_count++;
		printf("%lu\r", file_count);
	}

	closedir(dir);
	printf("%s done searching %s\n",timestr(), path);
	return(0);
}

int compare_files(const file_t *f1, const file_t *f2 )
{
	int result;
	
	/* Sort first by size (descending) */
	if((result = f2->length - f1->length) != 0)
		return(result);

	/* Then by chksum (ascending) */
	if((result = memcmp(&f1->chksum, &f2->chksum, sizeof(f1->chksum))) != 0)
		return(result);

	/* Then by date (descending) */
	return(f2->date - f1->date);
}

int main(int argc, char** argv)
{
	char hex[32];
	int i;
	ulong fsize;
	ulong dupe_count=0;
	ulong del_files=0;
	ulong del_bytes=0;
	ulong compare_bytes=0;
	BOOL recursive=FALSE;
	BOOL del_dupes=FALSE;
	BOOL dir_specified=FALSE;

	for(i=1;i<argc;i++) {
		if(!stricmp(argv[i],"-d"))
			del_dupes=TRUE;
		else if(!stricmp(argv[i],"-r"))
			recursive=TRUE;
		else if(!stricmp(argv[i],"-b") && i<argc+1)
			compare_bytes=atoi(argv[++i]);
		else if(!stricmp(argv[i],"-k") && i<argc+1)
			compare_bytes=atoi(argv[++i])*1024;
		else if(argv[i][0]=='-') {
			printf("%s [[-opt] [-opt] [...]] [[path] [path] [...]]\n", argv[0]);
			printf("-r\t search directories recursively\n");
			printf("-d\t delete duplicate files found\n");
			printf("-b n\t compare up to n bytes of each file\n");
			printf("-k n\t compare up to n kilobytes each of file\n");
			exit(0);
		}
		else {
			dir_specified=TRUE;
			searchdir(argv[i], recursive, compare_bytes);
		}
	}
	if(!dir_specified)
		searchdir(".", recursive, compare_bytes);

	if(!file_count) {
		printf("no files.\n");
		return(0);
	}

	printf("%s begin sorting (%lu files)\n", timestr(), file_count);
	qsort(file,file_count,sizeof(file_t),compare_files);
	printf("%s end sorting\n", timestr());

	printf("%s comparing (%lu files)\n", timestr(), file_count);

	for(i=0;i<file_count-1;i++) {
		if(file[i].length != file[i+1].length)
			continue; /* sizes must match */
		if(memcmp(&file[i].chksum, &file[i+1].chksum, sizeof(file[i].chksum)))
			continue; /* chksums must match */
#ifdef USE_MD5
		MD5_hex(hex, file[i].chksum);
#else
		sprintf(hex, "%08lx", file[i].chksum);
#endif
		printf("Dupe: %s %7lu %s\n", hex, file[i].length, getfname(file[i].path));
		if(del_dupes) {
			fsize=flength(file[i].path);
			printf("Removing %s (%lu bytes)\n", file[i].path, fsize);
			if(remove(file[i].path)!=0)
				perror(file[i].path);
			else {
				del_files++;
				del_bytes+=fsize;
			}
		}
		dupe_count++;
	}

	printf("%s done (%lu duplicates found)\n", timestr(), dupe_count);
	if(del_files)
		printf("%lu bytes deleted in %lu files\n", del_bytes, del_files);
	return(0);
}