? gdbm.patch Index: Makefile.am =================================================================== RCS file: /var/cvs/GNUnet/Makefile.am,v retrieving revision 1.8 diff -u -r1.8 Makefile.am --- Makefile.am 25 Mar 2002 08:46:19 -0000 1.8 +++ Makefile.am 4 May 2002 18:59:00 -0000 @@ -1,2 +1,3 @@ INCLUDES = -I$(top_srcdir)/src/include +LIBS = -lgdbm SUBDIRS = src doc Index: configure.in =================================================================== RCS file: /var/cvs/GNUnet/configure.in,v retrieving revision 1.26 diff -u -r1.26 configure.in --- configure.in 1 Apr 2002 08:19:36 -0000 1.26 +++ configure.in 4 May 2002 18:59:01 -0000 @@ -51,6 +51,7 @@ AC_CHECK_LIB(pthread, pthread_create) AC_CHECK_LIB(nsl, gethostbyname) AC_CHECK_LIB(socket, socket) +AC_CHECK_LIB(gdbm, gdbm_open) AM_PATH_GTK(1.2.0,,AC_MSG_ERROR(GNUnet needs GTK)) AM_PATH_GLIB(1.2.0,,AC_MSG_ERROR(GNUnet needs glib),gthread) Index: src/Makefile.am =================================================================== RCS file: /var/cvs/GNUnet/src/Makefile.am,v retrieving revision 1.9 diff -u -r1.9 Makefile.am --- src/Makefile.am 14 Mar 2002 09:27:49 -0000 1.9 +++ src/Makefile.am 4 May 2002 18:59:02 -0000 @@ -1,2 +1,3 @@ INCLUDES = -I$(top_srcdir)/src/include +LIBS = -lgdbm SUBDIRS = util common server textui test gtkui Index: src/common/lookup.c =================================================================== RCS file: /var/cvs/GNUnet/src/common/lookup.c,v retrieving revision 1.65 diff -u -r1.65 lookup.c --- src/common/lookup.c 15 Apr 2002 03:34:22 -0000 1.65 +++ src/common/lookup.c 4 May 2002 18:59:03 -0000 @@ -125,39 +125,26 @@ /* ******************* CODE... ****************** */ static void scanDataContent() { - DIR * dinfo; + DirName *dir; + GDBM_FILE dbf; + datum prevkey, key; ContentIndex entry; - FileName dirName; - struct stat istat; - struct dirent *finfo; - /* scan directory data/content and add entries to database + /* scan database data/content.dbf and add entries to database if not already present */ LOOKUP_filesStored_ = 0; LOOKUP_lips_db_index_ = rand() % LOOKUP_LIPSPART_COUNT; - dirName = expandFileName((FileName)getContentDirectory()); - stat((char*)dirName,&istat); - if (!S_ISDIR(istat.st_mode)) { -#ifdef PRINT_WARNINGS - print("contentDirectory must be invoked on a directory (%s)!\n", - dirName); -#endif - xfree(dirName,"scanDataContent expanded contentDirectory (1)"); - return; - } - errno = 0; - dinfo = opendir((char*)dirName); - if ((errno == EACCES)||(dinfo == NULL)) { -#ifdef PRINT_WARNINGS - print("initLookup: access denied (%s)\n",dirName); -#endif - xfree(dirName,"scanDataContent expanded contentDirectory (2)"); + dir = (DirName*)expandFileName((FileName)getContentDirectory()); + dbf = getDatabase(dir); + xfree(dir,"scanDataContent expanded contentDirectory (2)"); + if (!dbf) { return; } - while ((finfo = readdir(dinfo)) != NULL) { + key = gdbm_firstkey(dbf); + while (key.dptr) { LOOPPRINT("scanDataContent"); - if (strlen(finfo->d_name) == sizeof(HashCode160)*2) { - hex2hash((HexName*)finfo->d_name, + if (strlen(key.dptr) == sizeof(HashCode160)*2) { + hex2hash((HexName*)key.dptr, &entry.doubleHash); entry.importance = 0; entry.fileNameIndex = -1; /* directly available in data/content */ @@ -165,8 +152,11 @@ addEntry(&entry); LOOKUP_filesStored_++; /* one more file */ } + prevkey = key; + key = gdbm_nextkey(dbf, prevkey); + free(prevkey.dptr); } - closedir(dinfo); + gdbm_close(dbf); LOOKUP_lips_count_ = 0; } @@ -253,27 +243,19 @@ #endif hash2hex(query, &fil); - fn = xmalloc(strlen((char*)LOOKUP_contentDirectory_)+ - sizeof(HexName)+1, - "localLookup filename building"); - buildFileName(LOOKUP_contentDirectory_, - &fil, - fn); - j = readFile(fn, MAX_FSIZE * CONTENT_SIZE, buf); + j = readContent(&fil, MAX_FSIZE * CONTENT_SIZE, buf, LOOKUP_contentDirectory_); if (j <= 0) { #ifdef PRINT_QUERY - print("QUERY: local lookup: %s not found!\n",fn); + print("QUERY: local lookup: %s not found!\n",query); #endif - xfree(fn,"localLookup filename building (1)"); return 0; } /* data found on local node! */ if ((j % CONTENT_SIZE) != 0) { #ifdef PRINT_WARNINGS - print("Bad content file: %s\n",fn); + print("Bad content file: %s\n",query); #endif - xfree(fn,"localLookup filename building (2)" ); return 0; } #ifdef PRINT_QUERY @@ -462,14 +444,7 @@ /* remove the file */ hash2hex(entry, &fil); - fn = xmalloc(strlen((char*)LOOKUP_contentDirectory_)+ - sizeof(HexName)+1, - "deleteEntry: filename"); - buildFileName(LOOKUP_contentDirectory_, - &fil, - fn); - unlink(fn); - xfree(fn,"deleteEntrz contentDirectory expanded filename"); + unlinkFromDB(fil, LOOKUP_contentDirectory_); } /** Index: src/include/util/storage.h =================================================================== RCS file: /var/cvs/GNUnet/src/include/util/storage.h,v retrieving revision 1.32 diff -u -r1.32 storage.h --- src/include/util/storage.h 26 Mar 2002 06:24:17 -0000 1.32 +++ src/include/util/storage.h 4 May 2002 18:59:04 -0000 @@ -34,6 +34,7 @@ #ifndef SOMEBSD #include #endif +#include /* size of the blocks we slice file data into (and indirection node size) */ #define CONTENT_SIZE 1024 @@ -126,6 +127,12 @@ FileName result); /** + * Open a gdbm database (for content) + * @param dir the directory where content is configured to be stored (e.g. data/content). A file called ${dir}.dbf is used instead + **/ +GDBM_FILE getDatabase(DirName *dir); + +/** * Read the contents of a binary file into a buffer. * @param fileName the name of the file, not freed, * must already be expanded! @@ -136,6 +143,18 @@ int readFile(FileName fileName, int len, void * result); + +/** + * Read the contents of a bucket to a buffer. + * @param fileName the name of the file, not freed! + * @param len the maximum number of bytes to read + * @param result the buffer to write the result to + * @return the number of bytes read on success, -1 on failure + **/ +int readContent(HexName *fn, + int len, + void * result, + DirName *dir); /** * Write a buffer to a file. Index: src/server/policy.c =================================================================== RCS file: /var/cvs/GNUnet/src/server/policy.c,v retrieving revision 1.9 diff -u -r1.9 policy.c --- src/server/policy.c 18 Mar 2002 11:44:56 -0000 1.9 +++ src/server/policy.c 4 May 2002 18:59:04 -0000 @@ -168,3 +168,4 @@ /* end of policy.c */ + Index: src/util/storage.c =================================================================== RCS file: /var/cvs/GNUnet/src/util/storage.c,v retrieving revision 1.76 diff -u -r1.76 storage.c --- src/util/storage.c 4 May 2002 17:51:47 -0000 1.76 +++ src/util/storage.c 4 May 2002 18:59:05 -0000 @@ -162,6 +162,55 @@ } /** + * Open a gdbm database (for content) + * @param dir the directory where content is configured to be stored (e.g. data/content). A file called ${dir}.dbf is used instead + **/ +GDBM_FILE getDatabase(DirName *dir) { + FileName fin, ff; + GDBM_FILE dbf; + + char *ext = ".gdb"; + int fnSize = strlen(dir->dirName) + strlen(ext); + + fprintf(stderr, + "Database: %s\n", + dir->dirName); + + ff = xmalloc(fnSize, + "getDatabase: filaname"); + + if ( ff == NULL ) { + fprintf(stderr, + "getDatabase: out of memory\n"); + return NULL; /* failed! */ + } + + strcpy(ff, dir->dirName); + ff[strlen(ff) - 1] = 0; + strcat(ff, ext); + fin = expandFileName(ff); + xfree(ff, + "getDatabase: filaname"); + + dbf = gdbm_open(fin, 0, GDBM_WRCREAT, S_IRUSR|S_IWUSR, 0); + xfree(fin, + "getDatabase: filaname"); + + if (!dbf) { + fprintf(stderr, + "getDatabase: failed to open database file with error: %s", + gdbm_strerror(gdbm_errno)); + if (errno) { + perror(", system error "); + } else { + fprintf(stderr, + "\n"); + } + } + return(dbf); +} + +/** * Read the contents of a binary file into a buffer. * @param fileName the name of the file, not freed, * must already be expanded! @@ -187,6 +236,48 @@ } /** + * Read the contents of a bucket to a buffer. + * @param fileName the name of the file, not freed! + * @param len the maximum number of bytes to read + * @param result the buffer to write the result to + * @return the number of bytes read on success, -1 on failure + **/ +int readContent(HexName *fn, + int len, + void * result, + DirName *dir) { + + GDBM_FILE dbf; + datum key, buffer; + + if ((fn == NULL) || (result == NULL)) + return -1; + dbf = getDatabase(dir); + if ( !dbf ) { + return; + } + + key.dptr = fn->data; + key.dsize = strlen(key.dptr) + 1; + + buffer = gdbm_fetch(dbf, key); + if (!buffer.dptr) { + gdbm_close(dbf); + return -1; + } + if (buffer.dsize > len) { + free(buffer.dptr); + gdbm_close(dbf); + return -1; + } + + memcpy(result, buffer.dptr, buffer.dsize); + free(buffer.dptr); + gdbm_close(dbf); + return buffer.dsize; +} + +/** * Write a buffer to a file. * @param fileName the name of the file, NOT freed! * @param buffer the data to write @@ -211,6 +302,7 @@ close(handle); } + /** * Write content to a file. Check for reduncancy and eventually * append. @@ -221,70 +313,99 @@ CONTENT_Block * block, DirName * dir) { CONTENT_Block preBlock[MAX_FSIZE]; - FileName fin; - FileName ff; /* temporary */ + datum buffer, key; int i; int j; - int coll; + int missmatch; int blocks; - int handle; + GDBM_FILE dbf; - ff = xmalloc(strlen((char*)dir)+sizeof(HexName)+1, - "writeContent: filaname"); - if ( ff == NULL ) - return; /* failed! */ - - buildFileName(dir, fn, ff); - fin = expandFileName(ff); - xfree(ff,"writeContent: builded FN"); - handle = open(fin, O_RDWR|O_CREAT, S_IRUSR|S_IWUSR); - if (handle == -1) { - fprintf(stderr, - "Warning: failed to open file %s\n", fin); - xfree(fin,"writeContent: filaname (1)"); - return; /* failed! */ - } - xfree(fin, "writeContent: filaname (2)"); - j = read(handle, preBlock, MAX_FSIZE*CONTENT_SIZE); - if ((j == 0)||(j % CONTENT_SIZE != 0)) { - if (j !=0) { + dbf = getDatabase(dir); + if ( !dbf ) { + return; + } + + key.dptr = fn->data; + key.dsize = strlen(key.dptr) + 1; + buffer = gdbm_fetch(dbf, key); + if (!buffer.dptr) { + j = 0; + } else { + j = buffer.dsize; + memcpy(preBlock->content, buffer.dptr, j); + free(buffer.dptr); + } + + if ((!j) || (j % CONTENT_SIZE != 0)) { + if (!j) { fprintf(stderr, - "WARNING: file in content directory is corrupted!\n"); - if (0 != lseek(handle, 0, SEEK_SET)) { - fprintf(stderr, - "WARNING: lseek on corrupted file failed!\n"); - } - if (0 != ftruncate(handle, 0)) { - fprintf(stderr, - "WARNING: truncate on corrupted file failed!\n"); - } + "Info: New Bucket created\n", + MAX_FSIZE); + } else { + fprintf(stderr, + "WARNING: bucket in content database is corrupted!\n"); } - write(handle, block, CONTENT_SIZE); - close(handle); + buffer.dptr = block->content; + buffer.dsize = CONTENT_SIZE; + gdbm_store(dbf, key, buffer, GDBM_REPLACE); + gdbm_close(dbf); return; } blocks = j / CONTENT_SIZE; + fprintf(stderr, + "Info: Found bucket %s with %d blocks\n", + fn->data, blocks); if (blocks >= MAX_FSIZE) { fprintf(stderr, "Attention: MAX_FSIZE (%d) collisions reached for a keyword!\n", MAX_FSIZE); - close(handle); + gdbm_close(dbf); return; /* full! */ } /* collision! check if duplicate and if not, append */ for (i=0;icontent[j]) - coll++; - if (coll == 0) { - close(handle); + if (preBlock[i].content[j] != block->content[j]) { + missmatch++; + } + if (missmatch == 0) { + fprintf(stderr, + "Info: Data already in bucket.\n"); + gdbm_close(dbf); return; /* collision! */ } } - write(handle, block, CONTENT_SIZE); - close(handle); + fprintf(stderr, + "Info: Appended Data as block %d to bucket.\n", + blocks + 1); + memcpy(preBlock[blocks].content , block->content, CONTENT_SIZE); + buffer.dptr = preBlock->content; + buffer.dsize += CONTENT_SIZE; + gdbm_store(dbf, key, buffer, GDBM_REPLACE); + gdbm_close(dbf); +} + +/** + * Free space in the database by removing one file + * @param fn the name of the file (without directory) + **/ +void unlinkFromDB(HexName *fn, + DirName *dir) { + + GDBM_FILE dbf; + datum key, buffer; + + dbf = getDatabase(dir); + if ( !dbf ) { + return; + } + + key.dptr = fn->data; + key.dsize = strlen(key.dptr) + 1; + gdbm_delete(dbf, key); + } /**