From: Kristian Høgsberg Date: Fri, 3 Jul 2009 18:06:29 +0000 (+0100) Subject: Go back to having all info in one rzdb file. X-Git-Tag: 0.1~5 X-Git-Url: http://project.juiblex.co.uk/git/?a=commitdiff_plain;h=3485ee6cb08bb652ac54655bef7ffb5e0ae02f38;p=razor2.git%2F.git Go back to having all info in one rzdb file. We can still split the rzdb file into a main file and a file data and a details file, but that's only for optimizing the required download size. On the system we always combine the parts back into one rzdb file once downloaded. --- diff --git a/librazor/razor-internal.h b/librazor/razor-internal.h index 31a6b5e..cfc4328 100644 --- a/librazor/razor-internal.h +++ b/librazor/razor-internal.h @@ -37,8 +37,8 @@ #endif #define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) -#define ALIGN(value, base) (((value) + (base - 1)) & ~((base) - 1)) - +#define PADDING(value, base) (-(value) & (base - 1)) +#define ALIGN(value, base) ((value) + PADDING(value, base)) void *zalloc(size_t size); struct razor_set_section { @@ -118,15 +118,7 @@ struct razor_set { struct array prefix_pool; struct array file_string_pool; struct array details_string_pool; - - struct razor_set_header *header; - size_t header_size; - - struct razor_set_header *details_header; - size_t details_header_size; - - struct razor_set_header *files_header; - size_t files_header_size; + struct razor_mapped_file *mapped_files; }; struct import_entry { diff --git a/librazor/razor.c b/librazor/razor.c index 3fac1f2..434c1bf 100644 --- a/librazor/razor.c +++ b/librazor/razor.c @@ -59,25 +59,27 @@ zalloc(size_t size) struct razor_set_section_index { const char *name; uint32_t offset; + uint32_t flags; }; -struct razor_set_section_index razor_sections[] = { - { RAZOR_STRING_POOL, offsetof(struct razor_set, string_pool) }, - { RAZOR_PACKAGES, offsetof(struct razor_set, packages) }, - { RAZOR_PROPERTIES, offsetof(struct razor_set, properties) }, - { RAZOR_PACKAGE_POOL, offsetof(struct razor_set, package_pool) }, - { RAZOR_PROPERTY_POOL, offsetof(struct razor_set, property_pool) }, - { RAZOR_PREFIX_POOL, offsetof(struct razor_set, prefix_pool) }, -}; - -struct razor_set_section_index razor_files_sections[] = { - { RAZOR_FILES, offsetof(struct razor_set, files) }, - { RAZOR_FILE_POOL, offsetof(struct razor_set, file_pool) }, - { RAZOR_FILE_STRING_POOL, offsetof(struct razor_set, file_string_pool) }, -}; +#define MAIN(type, field) \ + { type, offsetof(struct razor_set, field), RAZOR_SECTION_MAIN } +#define FILES(type, field) \ + { type, offsetof(struct razor_set, field), RAZOR_SECTION_FILES } +#define DETAILS(type, field) \ + { type, offsetof(struct razor_set, field), RAZOR_SECTION_DETAILS } -struct razor_set_section_index razor_details_sections[] = { - { RAZOR_DETAILS_STRING_POOL, offsetof(struct razor_set, details_string_pool) }, +struct razor_set_section_index razor_sections[] = { + MAIN(RAZOR_STRING_POOL, string_pool), + MAIN(RAZOR_PACKAGES, packages), + MAIN(RAZOR_PROPERTIES, properties), + MAIN(RAZOR_PACKAGE_POOL, package_pool), + MAIN(RAZOR_PROPERTY_POOL, property_pool), + MAIN(RAZOR_PREFIX_POOL, prefix_pool), + FILES(RAZOR_FILES, files), + FILES(RAZOR_FILE_POOL, file_pool), + FILES(RAZOR_FILE_STRING_POOL, file_string_pool), + DETAILS(RAZOR_DETAILS_STRING_POOL, details_string_pool) }; RAZOR_EXPORT struct razor_set * @@ -111,37 +113,47 @@ razor_set_create(void) return set; } -static int -razor_set_bind_sections(struct razor_set *set, - struct razor_set_header **header, - size_t *header_size, - struct razor_set_section_index section_index[], - int section_index_size, - const char *filename) +struct razor_mapped_file { + struct razor_set_header *header; + size_t size; + struct razor_mapped_file *next; +}; + +RAZOR_EXPORT int +razor_set_bind_sections(struct razor_set *set, const char *filename) { struct razor_set_section *s, *sections; - struct array *array; + struct razor_mapped_file *file; const char *pool; - int i; + struct array *array; + int i, j; - *header = razor_file_get_contents(filename, header_size); - if (!*header) + file = zalloc(sizeof *file); + if (file == NULL) return -1; - sections = (void *) *header + sizeof **header; - pool = (void *) sections + (*header)->num_sections * sizeof *sections; + file->header = razor_file_get_contents(filename, &file->size); + if (!file->header) { + free(file); + return -1; + } - for (i = 0; i < (*header)->num_sections; i++) { - int j; + file->next = set->mapped_files; + set->mapped_files = file; + + sections = (void *) file->header + sizeof *file->header; + pool = (void *) sections + + file->header->num_sections * sizeof *sections; + + for (i = 0; i < file->header->num_sections; i++) { s = sections + i; - for (j = 0; j < section_index_size; j++) - if (!strcmp(section_index[j].name, - &pool[s->name])) + for (j = 0; j < ARRAY_SIZE(razor_sections); j++) + if (!strcmp(razor_sections[j].name, &pool[s->name])) break; - if (j == section_index_size) + if (j == ARRAY_SIZE(razor_sections)) continue; - array = (void *) set + section_index[j].offset; - array->data = (void *) *header + s->offset; + array = (void *) set + razor_sections[j].offset; + array->data = (void *) file->header + s->offset; array->size = s->size; array->alloc = s->size; } @@ -155,149 +167,93 @@ razor_set_open(const char *filename) struct razor_set *set; set = zalloc(sizeof *set); - if (razor_set_bind_sections(set, &set->header, &set->header_size, - razor_sections, ARRAY_SIZE(razor_sections), - filename)){ + if (razor_set_bind_sections(set, filename)){ free(set); return NULL; } return set; } -RAZOR_EXPORT int -razor_set_open_details(struct razor_set *set, const char *filename) -{ - return razor_set_bind_sections(set, &set->details_header, - &set->details_header_size, - razor_details_sections, - ARRAY_SIZE(razor_details_sections), - filename); -} - -RAZOR_EXPORT int -razor_set_open_files(struct razor_set *set, const char *filename) -{ - return razor_set_bind_sections(set, &set->files_header, - &set->files_header_size, - razor_files_sections, - ARRAY_SIZE(razor_files_sections), - filename); -} - RAZOR_EXPORT void razor_set_destroy(struct razor_set *set) { - struct array *a; + struct razor_mapped_file *file, *next; + struct array *array; int i; assert (set != NULL); - if (set->header) { - razor_file_free_contents(set->header, set->header_size); - } else { + if (set->mapped_files == NULL) { for (i = 0; i < ARRAY_SIZE(razor_sections); i++) { - a = (void *) set + razor_sections[i].offset; - free(a->data); + array = (void *) set + razor_sections[i].offset; + array_release(array); } - } - - if (set->details_header) { - razor_file_free_contents(set->details_header, - set->details_header_size); - } else { - for (i = 0; i < ARRAY_SIZE(razor_details_sections); i++) { - a = (void *) set + razor_details_sections[i].offset; - free(a->data); - } - } - - if (set->files_header) { - razor_file_free_contents(set->files_header, - set->files_header_size); } else { - for (i = 0; i < ARRAY_SIZE(razor_files_sections); i++) { - a = (void *) set + razor_files_sections[i].offset; - free(a->data); + for (file = set->mapped_files; file != NULL; file = next) { + next = file->next; + razor_file_free_contents(file->header, file->size); + free(file); } } free(set); } -static int -razor_set_write_sections_to_fd(struct razor_set *set, int fd, - struct razor_set_section_index *sections, - size_t array_size) +RAZOR_EXPORT int +razor_set_write_to_fd(struct razor_set *set, int fd, uint32_t section_mask) { struct razor_set_header header; - struct razor_set_section *out_sections = - malloc(array_size * sizeof *out_sections); + struct razor_set_section sections[ARRAY_SIZE(razor_sections)]; struct hashtable table; - struct array *a, pool; + struct array pool, *arrays[ARRAY_SIZE(razor_sections)]; uint32_t offset; - int i; - - header.magic = RAZOR_MAGIC; - header.version = RAZOR_VERSION; - header.num_sections = array_size; - offset = sizeof header + array_size * sizeof *out_sections; + int count, i, j; + static const char padding[4]; array_init(&pool); hashtable_init(&table, &pool); - for (i = 0; i < array_size; i++) - out_sections[i].name = - hashtable_tokenize(&table, sections[i].name); + j = 0; + for (i = 0; i < ARRAY_SIZE(razor_sections); i++) { + if ((razor_sections[i].flags & section_mask) == 0) + continue; + + arrays[j] = (void *) set + razor_sections[i].offset; + sections[j].name = + hashtable_tokenize(&table, razor_sections[i].name); + j++; + } - offset += pool.size; + count = j; + header.magic = RAZOR_MAGIC; + header.version = RAZOR_VERSION; + header.num_sections = count; + offset = sizeof header + count * sizeof *sections + ALIGN(pool.size, 4); - for (i = 0; i < array_size; i++) { - a = (void *) set + sections[i].offset; - out_sections[i].offset = offset; - out_sections[i].size = a->size; - offset += a->size; + for (i = 0; i < count; i++) { + sections[i].offset = offset; + sections[i].size = arrays[i]->size; + offset += ALIGN(arrays[i]->size, 4); } razor_write(fd, &header, sizeof header); - razor_write(fd, out_sections, array_size * sizeof *out_sections); + razor_write(fd, sections, count * sizeof *sections); razor_write(fd, pool.data, pool.size); + razor_write(fd, padding, PADDING(pool.size, 4)); - for (i = 0; i < array_size; i++) { - a = (void *) set + sections[i].offset; - razor_write(fd, a->data, a->size); + for (i = 0; i < count; i++) { + razor_write(fd, arrays[i]->data, arrays[i]->size); + razor_write(fd, padding, PADDING(arrays[i]->size, 4)); } - free(out_sections); + array_release(&pool); + hashtable_release(&table); return 0; } RAZOR_EXPORT int -razor_set_write_to_fd(struct razor_set *set, int fd, - enum razor_repo_file_type type) -{ - switch (type) { - case RAZOR_REPO_FILE_MAIN: - return razor_set_write_sections_to_fd(set, fd, - razor_sections, - ARRAY_SIZE(razor_sections)); - - case RAZOR_REPO_FILE_DETAILS: - return razor_set_write_sections_to_fd(set, fd, - razor_details_sections, - ARRAY_SIZE(razor_details_sections)); - case RAZOR_REPO_FILE_FILES: - return razor_set_write_sections_to_fd(set, fd, - razor_files_sections, - ARRAY_SIZE(razor_files_sections)); - default: - return -1; - } -} - -RAZOR_EXPORT int -razor_set_write(struct razor_set *set, const char *filename, - enum razor_repo_file_type type) +razor_set_write(struct razor_set *set, const char *filename, uint32_t sections) { int fd, status; @@ -305,7 +261,7 @@ razor_set_write(struct razor_set *set, const char *filename, if (fd < 0) return -1; - status = razor_set_write_to_fd(set, fd, type); + status = razor_set_write_to_fd(set, fd, sections); if (status) { close(fd); return status; diff --git a/librazor/razor.h b/librazor/razor.h index f63b96b..19e732c 100644 --- a/librazor/razor.h +++ b/librazor/razor.h @@ -23,10 +23,11 @@ #include -enum razor_repo_file_type { - RAZOR_REPO_FILE_MAIN, - RAZOR_REPO_FILE_DETAILS, - RAZOR_REPO_FILE_FILES +enum razor_section_type { + RAZOR_SECTION_MAIN = 0x01, + RAZOR_SECTION_DETAILS = 0x02, + RAZOR_SECTION_FILES = 0x04, + RAZOR_SECTION_ALL = 0x07 }; enum razor_detail_type { @@ -95,13 +96,11 @@ struct razor_set *razor_set_create_without_root(void); struct razor_set *razor_set_create(void); struct razor_set *razor_set_open(const char *filename); void razor_set_destroy(struct razor_set *set); -int razor_set_write_to_fd(struct razor_set *set, int fd, - enum razor_repo_file_type type); -int razor_set_write(struct razor_set *set, const char *filename, - enum razor_repo_file_type type); - -int razor_set_open_details(struct razor_set *set, const char *filename); -int razor_set_open_files(struct razor_set *set, const char *filename); +int razor_set_write_to_fd(struct razor_set *set, + int fd, uint32_t section_mask); +int razor_set_write(struct razor_set *set, + const char *filename, uint32_t setions); +int razor_set_bind_sections(struct razor_set *set, const char *filename); struct razor_package * razor_set_get_package(struct razor_set *set, const char *package); diff --git a/librazor/root.c b/librazor/root.c index 13bfceb..d80cbec 100644 --- a/librazor/root.c +++ b/librazor/root.c @@ -42,9 +42,6 @@ #endif static const char system_repo_filename[] = "system.rzdb"; -static const char system_repo_details_filename[] = "system-details.rzdb"; -static const char system_repo_files_filename[] = "system-files.rzdb"; - static const char next_repo_filename[] = "system-next.rzdb"; #ifdef MSWIN_API #define RAZOR_ROOT_PATH NULL @@ -57,7 +54,6 @@ struct razor_root { struct razor_set *system; struct razor_set *next; int fd; - char root[PATH_MAX]; char path[PATH_MAX]; char new_path[PATH_MAX]; }; @@ -82,7 +78,7 @@ razor_root_create(const char *root) { struct stat buf; struct razor_set *set; - char path[PATH_MAX], details_path[PATH_MAX], files_path[PATH_MAX]; + char path[PATH_MAX]; assert (root != NULL); @@ -115,18 +111,12 @@ razor_root_create(const char *root) set = razor_set_create(); snprintf(path, sizeof path, "%s%s/%s", root, razor_root_path, system_repo_filename); - snprintf(details_path, sizeof details_path, "%s%s/%s", - root, razor_root_path, system_repo_details_filename); - snprintf(files_path, sizeof files_path, "%s%s/%s", - root, razor_root_path, system_repo_files_filename); if (stat(path, &buf) == 0) { fprintf(stderr, "a razor install root is already initialized\n"); return -1; } - if (razor_set_write(set, path, RAZOR_REPO_FILE_MAIN) < 0 || - razor_set_write(set, details_path, RAZOR_REPO_FILE_DETAILS) < 0 || - razor_set_write(set, files_path, RAZOR_REPO_FILE_FILES) < 0 ) { + if (razor_set_write(set, path, RAZOR_SECTION_ALL) < 0) { fprintf(stderr, "could not write initial package set\n"); return -1; } @@ -139,7 +129,6 @@ RAZOR_EXPORT struct razor_root * razor_root_open(const char *root) { struct razor_root *image; - char details_path[PATH_MAX], files_path[PATH_MAX]; assert (root != NULL); @@ -169,19 +158,9 @@ razor_root_open(const char *root) snprintf(image->path, sizeof image->path, "%s%s/%s", root, razor_root_path, system_repo_filename); - snprintf(details_path, sizeof details_path, - "%s%s/%s", root, razor_root_path, system_repo_details_filename); - snprintf(files_path, sizeof files_path, - "%s%s/%s", root, razor_root_path, system_repo_files_filename); - - /* FIXME: We store the root path to make the hack in - * razor_root_update() work. Need to get rid of this. */ - strcpy(image->root, root); image->system = razor_set_open(image->path); - if (image->system == NULL || - razor_set_open_details(image->system, details_path) || - razor_set_open_files(image->system, files_path)) { + if (image->system == NULL) { unlink(image->new_path); close(image->fd); free(image); @@ -194,31 +173,15 @@ razor_root_open(const char *root) RAZOR_EXPORT struct razor_set * razor_root_open_read_only(const char *root) { - char path[PATH_MAX], details_path[PATH_MAX], files_path[PATH_MAX]; - struct razor_set *set; + char path[PATH_MAX]; assert (root != NULL); razor_root_init(); snprintf(path, sizeof path, "%s%s/%s", root, razor_root_path, system_repo_filename); - snprintf(details_path, sizeof details_path, - "%s%s/%s", root, razor_root_path, system_repo_details_filename); - snprintf(files_path, sizeof files_path, - "%s%s/%s", root, razor_root_path, system_repo_files_filename); - - - set = razor_set_open(path); - if (set == NULL) - return NULL; - - if (razor_set_open_details(set, details_path) || - razor_set_open_files(set, files_path)) { - razor_set_destroy(set); - return NULL; - } - return set; + return razor_set_open(path); } RAZOR_EXPORT struct razor_set * @@ -245,27 +208,13 @@ razor_root_close(struct razor_root *root) RAZOR_EXPORT void razor_root_update(struct razor_root *root, struct razor_set *next) { - char path[PATH_MAX]; - assert (root != NULL); assert (next != NULL); razor_root_init(); - razor_set_write_to_fd(next, root->fd, RAZOR_REPO_FILE_MAIN); + razor_set_write_to_fd(next, root->fd, RAZOR_SECTION_ALL); root->next = next; - /* FIXME: This is a pretty bad hack that just overwrites the - * system details and files rzdb files before the transaction - * succeeds. We need to fix this by merging the separate - * details and files rzdb files back into the main rzdb - * file. */ - snprintf(path, sizeof path, - "%s%s/%s", root->root, razor_root_path, system_repo_details_filename); - razor_set_write(next, path, RAZOR_REPO_FILE_DETAILS); - snprintf(path, sizeof path, - "%s%s/%s", root->root, razor_root_path, system_repo_files_filename); - razor_set_write(next, path, RAZOR_REPO_FILE_FILES); - /* Sync the new repo file so the new package set is on disk * before we start upgrading. */ fsync(root->fd); diff --git a/src/main.c b/src/main.c index 43a6612..28eddc8 100644 --- a/src/main.c +++ b/src/main.c @@ -471,19 +471,10 @@ command_import_yum(int argc, const char *argv[]) set = razor_set_create_from_yum(); if (set == NULL) return 1; - if (razor_set_write(set, rawhide_repo_filename, RAZOR_REPO_FILE_MAIN)) { + if (razor_set_write(set, rawhide_repo_filename, RAZOR_SECTION_ALL)) { perror(rawhide_repo_filename); return -1; } - if (razor_set_write(set, "rawhide-details.rzdb", - RAZOR_REPO_FILE_DETAILS)) { - perror("rawhide-details.rzdb"); - return -1; - } - if (razor_set_write(set, "rawhide-files.rzdb", RAZOR_REPO_FILE_FILES)) { - perror("rawhide-files.rzdb"); - return -1; - } razor_set_destroy(set); printf("wrote %s\n", rawhide_repo_filename); @@ -569,9 +560,7 @@ command_update(int argc, const char *argv[]) return 1; upstream = razor_set_open(rawhide_repo_filename); - if (upstream == NULL || - razor_set_open_details(upstream, "rawhide-details.rzdb") || - razor_set_open_files(upstream, "rawhide-files.rzdb")) + if (upstream == NULL) return 1; trans = razor_transaction_create(set, upstream); @@ -594,7 +583,7 @@ command_update(int argc, const char *argv[]) } set = razor_transaction_commit(trans); - razor_set_write(set, updated_repo_filename, RAZOR_REPO_FILE_MAIN); + razor_set_write(set, updated_repo_filename, RAZOR_SECTION_ALL); razor_transaction_destroy(trans); razor_set_destroy(set); razor_set_destroy(upstream); @@ -737,9 +726,7 @@ command_import_rpms(int argc, const char *argv[]) printf("\nsaving\n"); set = razor_importer_finish(importer); - razor_set_write(set, repo_filename, RAZOR_REPO_FILE_MAIN); - razor_set_write(set, "system-details.rzdb", RAZOR_REPO_FILE_DETAILS); - razor_set_write(set, "system-files.rzdb", RAZOR_REPO_FILE_FILES); + razor_set_write(set, repo_filename, RAZOR_SECTION_ALL); razor_set_destroy(set); printf("wrote %s\n", repo_filename); @@ -995,12 +982,10 @@ command_install(int argc, const char *argv[]) system = razor_root_get_system_set(root); upstream = razor_set_open(rawhide_repo_filename); - if (upstream == NULL || - razor_set_open_details(upstream, "rawhide-details.rzdb") || - razor_set_open_files(upstream, "rawhide-files.rzdb")) { - fprintf(stderr, "couldn't open rawhide repo\n"); - razor_root_close(root); - return 1; + if (upstream == NULL) { + fprintf(stderr, "couldn't open rawhide repo\n"); + razor_root_close(root); + return 1; } if (relocations) { @@ -1177,9 +1162,6 @@ command_search(int argc, const char *argv[]) if (set == NULL) return 1; - if (razor_set_open_details(set, "rawhide-details.rzdb")) - return 1; - pi = razor_package_iterator_create(set); while (razor_package_iterator_next(pi, &package, RAZOR_DETAIL_NAME, &name, diff --git a/src/rpm.c b/src/rpm.c index 225bb99..63b6ee0 100644 --- a/src/rpm.c +++ b/src/rpm.c @@ -340,7 +340,6 @@ get_query_packages(struct razor_set *set, int argc, const char *argv[]) { struct razor_package_query *query; struct razor_package_iterator *pi; - char *files; int i; if (option_all + option_whatprovides + option_whatrequires + @@ -350,11 +349,6 @@ get_query_packages(struct razor_set *set, int argc, const char *argv[]) exit(1); } - files = "install/var/lib/razor/system-files.rzdb"; - if (option_file) - if (razor_set_open_files(set, files)) - exit(1); - query = razor_package_query_create(set); if (option_all) { @@ -471,7 +465,7 @@ command_query(int argc, const char *argv[]) struct razor_set *set; struct razor_package_iterator *pi; struct razor_package *package; - const char *name, *version, *arch, *details, *files; + const char *name, *version, *arch; if (option_package) { set = create_set_from_command_line(argc, argv); @@ -479,16 +473,6 @@ command_query(int argc, const char *argv[]) option_all = 1; } else { set = razor_root_open_read_only(option_root); - - /* FIXME: We need to figure out how to do this right. */ - details = "install/var/lib/razor/system-details.rzdb"; - if (option_info) - if (razor_set_open_details(set, details)) - return; - files = "install/var/lib/razor/system-files.rzdb"; - if (option_list) - if (razor_set_open_files(set, files)) - return; } pi = get_query_packages(set, argc, argv);