Refactor razor_set_write() a bit and stop writing the buckets section.
22 array_init(struct array *array)
24 memset(array, 0, sizeof *array);
28 array_release(struct array *array)
34 array_add(struct array *array, int size)
44 while (alloc < array->size + size)
47 if (array->alloc < alloc) {
48 data = realloc(array->data, alloc);
55 p = array->data + array->size;
62 write_to_fd(int fd, void *p, size_t size)
68 len = write(fd, p, rest);
78 write_to_file(const char *filename, void *p, size_t size)
82 fd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0666);
85 err = write_to_fd(fd, p, size);
102 struct razor_set_header {
104 unsigned int version;
105 struct { unsigned int type, offset, size; } sections[0];
108 #define RAZOR_MAGIC 0x7a7a7a7a
109 #define RAZOR_VERSION 1
111 #define RAZOR_BUCKETS 1
112 #define RAZOR_STRINGS 2
113 #define RAZOR_PACKAGES 3
114 #define RAZOR_REQUIRES 4
115 #define RAZOR_PROVIDES 5
116 #define RAZOR_PROPERTIES 6
118 struct razor_package {
120 unsigned long version;
121 unsigned long requires;
122 unsigned long provides;
125 struct razor_property {
127 unsigned long version;
128 unsigned long packages;
132 struct array buckets;
133 struct array string_pool;
134 struct array property_pool;
135 struct array packages;
136 struct array requires;
137 struct array provides;
138 struct razor_set_header *header;
142 razor_set_create(void)
144 struct razor_set *set;
147 set = zalloc(sizeof(struct razor_set));
148 p = array_add(&set->string_pool, 1);
155 razor_set_open(const char *filename)
157 struct razor_set *set;
159 unsigned int size, offset;
162 set = zalloc(sizeof *set);
163 fd = open(filename, O_RDONLY);
164 if (fstat(fd, &stat) < 0)
166 set->header = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
167 if (set->header == MAP_FAILED) {
172 for (i = 0; i < set->header->sections[i].type; i++) {
173 offset = set->header->sections[i].offset;
174 size = set->header->sections[i].size;
176 switch (set->header->sections[i].type) {
178 set->string_pool.data = (void *) set->header + offset;
179 set->string_pool.size = size;
180 set->string_pool.alloc = size;
183 set->packages.data = (void *) set->header + offset;
184 set->packages.size = size;
185 set->packages.size = size;
188 set->requires.data = (void *) set->header + offset;
189 set->requires.size = size;
190 set->requires.size = size;
193 set->provides.data = (void *) set->header + offset;
194 set->provides.size = size;
195 set->provides.size = size;
197 case RAZOR_PROPERTIES:
198 set->property_pool.data = (void *) set->header + offset;
199 set->property_pool.size = size;
200 set->property_pool.size = size;
210 razor_set_destroy(struct razor_set *set)
216 for (i = 0; set->header->sections[i].type; i++)
218 size = set->header->sections[i].type;
219 munmap(set->header, size);
220 free(set->buckets.data);
222 free(set->buckets.data);
223 free(set->string_pool.data);
224 free(set->packages.data);
225 free(set->requires.data);
226 free(set->provides.data);
227 free(set->property_pool.data);
234 razor_set_write(struct razor_set *set, const char *filename)
237 struct razor_set_header *header = (struct razor_set_header *) data;
238 unsigned long offset;
240 struct { int type; struct array *array; } sections[] = {
241 { RAZOR_STRINGS, &set->string_pool },
242 { RAZOR_PACKAGES, &set->packages },
243 { RAZOR_REQUIRES, &set->requires },
244 { RAZOR_PROVIDES, &set->provides },
245 { RAZOR_PROPERTIES, &set->property_pool },
249 memset(data, 0, sizeof data);
250 header->magic = RAZOR_MAGIC;
251 header->version = RAZOR_VERSION;
252 offset = sizeof data;
254 for (i = 0; sections[i].type != 0; i++) {
255 header->sections[i].type = sections[i].type;
256 header->sections[i].offset = offset;
257 header->sections[i].size = sections[i].array->size;
258 offset += (sections[i].array->size + 4095) & ~4095;
261 header->sections[i].type = 0;
262 header->sections[i].offset = 0;
263 header->sections[i].size = 0;
265 fd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0666);
269 write_to_fd(fd, data, sizeof data);
270 for (i = 0; sections[i].type != 0; i++)
271 write_to_fd(fd, sections[i].array->data,
272 (sections[i].array->size + 4095) & ~4095);
280 hash_string(const char *key)
283 unsigned int hash = 0;
285 for (p = key; *p; p++)
286 hash = (hash * 617) ^ *p;
292 razor_set_lookup(struct razor_set *set, const char *key)
294 unsigned int mask, start, i;
298 pool = set->string_pool.data;
299 mask = set->buckets.alloc - 1;
300 start = hash_string(key) * sizeof(unsigned long);
302 for (i = 0; i < set->buckets.alloc; i += sizeof *b) {
303 b = set->buckets.data + ((start + i) & mask);
308 if (strcmp(key, &pool[*b]) == 0)
316 add_to_string_pool(struct razor_set *set, const char *key)
321 len = strlen(key) + 1;
322 p = array_add(&set->string_pool, len);
325 return p - (char *) set->string_pool.data;
329 add_to_property_pool(struct razor_set *set, struct array *properties)
333 p = array_add(properties, sizeof *p);
335 p = array_add(&set->property_pool, properties->size);
336 memcpy(p, properties->data, properties->size);
338 return p - (unsigned long *) set->property_pool.data;
342 do_insert(struct razor_set *set, unsigned long value)
344 unsigned int mask, start, i;
348 key = (char *) set->string_pool.data + value;
349 mask = set->buckets.alloc - 1;
350 start = hash_string(key) * sizeof(unsigned long);
352 for (i = 0; i < set->buckets.alloc; i += sizeof *b) {
353 b = set->buckets.data + ((start + i) & mask);
362 razor_set_insert(struct razor_set *set, const char *key)
364 unsigned long value, *buckets, *b, *end;
367 alloc = set->buckets.alloc;
368 array_add(&set->buckets, 4 * sizeof *buckets);
369 if (alloc != set->buckets.alloc) {
370 end = set->buckets.data + alloc;
371 memset(end, 0, set->buckets.alloc - alloc);
372 for (b = set->buckets.data; b < end; b++) {
376 do_insert(set, value);
381 value = add_to_string_pool(set, key);
382 do_insert (set, value);
388 razor_set_tokenize(struct razor_set *set, const char *string)
395 token = razor_set_lookup(set, string);
399 return razor_set_insert(set, string);
402 struct import_property_context {
404 struct array package;
407 struct import_context {
408 struct razor_set *set;
409 struct import_property_context requires;
410 struct import_property_context provides;
411 unsigned long package;
412 unsigned long *requires_map;
413 unsigned long *provides_map;
416 struct import_property {
418 unsigned long version;
419 unsigned long package;
421 unsigned long unique_index;
425 import_context_add_package(struct import_context *ctx,
426 const char *name, const char *version)
428 struct razor_package *p;
430 p = array_add(&ctx->set->packages, sizeof *p);
431 p->name = razor_set_tokenize(ctx->set, name);
432 p->version = razor_set_tokenize(ctx->set, version);
434 ctx->package = p - (struct razor_package *) ctx->set->packages.data;
435 array_init(&ctx->requires.package);
436 array_init(&ctx->provides.package);
440 import_context_finish_package(struct import_context *ctx)
442 struct razor_package *p;
444 p = (struct razor_package *) ctx->set->packages.data + ctx->package;
445 p->requires = add_to_property_pool(ctx->set, &ctx->requires.package);
446 p->provides = add_to_property_pool(ctx->set, &ctx->provides.package);
448 array_release(&ctx->requires.package);
449 array_release(&ctx->provides.package);
453 import_context_add_property(struct import_context *ctx,
454 struct import_property_context *pctx,
455 const char *name, const char *version)
457 struct import_property *p;
460 p = array_add(&pctx->all, sizeof *p);
461 p->name = razor_set_tokenize(ctx->set, name);
462 p->version = razor_set_tokenize(ctx->set, version);
463 p->package = ctx->package;
464 p->index = p - (struct import_property *) pctx->all.data;
466 r = array_add(&pctx->package, sizeof *r);
471 parse_package(struct import_context *ctx, const char **atts, void *data)
473 const char *name = NULL, *version = NULL;
476 for (i = 0; atts[i]; i += 2) {
477 if (strcmp(atts[i], "name") == 0)
479 else if (strcmp(atts[i], "version") == 0)
480 version = atts[i + 1];
483 if (name == NULL || version == NULL) {
484 fprintf(stderr, "invalid package tag, "
485 "missing name or version attributes\n");
489 import_context_add_package(ctx, name, version);
493 parse_property(struct import_context *ctx, const char **atts, void *data)
495 const char *name = NULL, *version = NULL;
498 for (i = 0; atts[i]; i += 2) {
499 if (strcmp(atts[i], "name") == 0)
501 if (strcmp(atts[i], "version") == 0)
502 version = atts[i + 1];
506 fprintf(stderr, "invalid tag, missing name attribute\n");
510 import_context_add_property(ctx, data, name, version);
514 start_element(void *data, const char *name, const char **atts)
516 struct import_context *ctx = data;
518 if (strcmp(name, "package") == 0)
519 parse_package(ctx, atts, NULL);
520 else if (strcmp(name, "requires") == 0)
521 parse_property(ctx, atts, &ctx->requires);
522 else if (strcmp(name, "provides") == 0)
523 parse_property(ctx, atts, &ctx->provides);
527 end_element (void *data, const char *name)
529 struct import_context *ctx = data;
531 if (strcmp(name, "package") == 0)
532 import_context_finish_package(ctx);
536 sha1_to_hex(const unsigned char *sha1)
539 static char hexbuffer[4][50];
540 static const char hex[] = "0123456789abcdef";
541 char *buffer = hexbuffer[3 & ++bufno], *buf = buffer;
544 for (i = 0; i < 20; i++) {
545 unsigned int val = *sha1++;
546 *buf++ = hex[val >> 4];
547 *buf++ = hex[val & 0xf];
555 razor_prepare_import(struct import_context *ctx)
557 memset(ctx, 0, sizeof *ctx);
558 ctx->set = razor_set_create();
562 razor_import(struct import_context *ctx, const char *filename)
570 unsigned char hash[20];
572 fd = open(filename, O_RDONLY);
573 if (fstat(fd, &stat) < 0)
575 p = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
579 parser = XML_ParserCreate(NULL);
580 XML_SetUserData(parser, ctx);
581 XML_SetElementHandler(parser, start_element, end_element);
582 if (XML_Parse(parser, p, stat.st_size, 1) == XML_STATUS_ERROR) {
584 "%s at line %d, %s\n",
585 XML_ErrorString(XML_GetErrorCode(parser)),
586 XML_GetCurrentLineNumber(parser),
591 XML_ParserFree(parser);
594 SHA1_Update(&sha1, p, stat.st_size);
595 SHA1_Final(hash, &sha1);
599 snprintf(buf, sizeof buf, "set/%s", sha1_to_hex(hash));
600 if (write_to_file(buf, p, stat.st_size) < 0)
602 munmap(p, stat.st_size);
607 static struct razor_set *qsort_set;
610 compare_packages(const void *p1, const void *p2)
612 const struct razor_package *pkg1 = p1, *pkg2 = p2;
613 char *pool = qsort_set->string_pool.data;
615 return strcmp(&pool[pkg1->name], &pool[pkg2->name]);
619 compare_properties(const void *p1, const void *p2)
621 const struct import_property *prop1 = p1, *prop2 = p2;
622 char *pool = qsort_set->string_pool.data;
625 result = strcmp(&pool[prop1->name], &pool[prop2->name]);
627 return strcmp(&pool[prop1->version], &pool[prop2->version]);
632 static unsigned long *
633 uniqueify_properties(struct array *in, struct array *out)
635 struct import_property *ip, *end;
636 struct razor_property *rp;
640 count = in->size / sizeof(struct import_property);
641 qsort(in->data, count,
642 sizeof(struct import_property), compare_properties);
645 end = in->data + in->size;
646 for (ip = in->data; ip < end; ip++) {
648 ip->name != rp->name || ip->version != rp->version) {
649 rp = array_add(out, sizeof *rp);
651 rp->version = ip->version;
653 ip->unique_index = rp - (struct razor_property *) out->data;
656 map = malloc(count * sizeof (unsigned long));
658 for (i = 0; i < count; i++)
659 map[ip[i].index] = ip[i].unique_index;
665 sort_packages(struct import_context *ctx)
667 struct razor_package *p, *end;
668 unsigned long *pool, *r;
670 pool = ctx->set->property_pool.data;
671 end = ctx->set->packages.data + ctx->set->packages.size;
672 for (p = ctx->set->packages.data; p < end; p++) {
673 for (r = &pool[p->requires]; *r; r++)
674 *r = ctx->requires_map[*r];
675 for (r = &pool[p->provides]; *r; r++)
676 *r = ctx->provides_map[*r];
679 qsort(ctx->set->packages.data,
680 ctx->set->packages.size / sizeof(struct razor_package),
681 sizeof(struct razor_package), compare_packages);
684 static struct razor_set *
685 razor_finish_import(struct import_context *ctx)
687 qsort_set = ctx->set;
690 uniqueify_properties(&ctx->requires.all, &ctx->set->requires);
692 uniqueify_properties(&ctx->provides.all, &ctx->set->provides);
696 free(ctx->requires.all.data);
697 free(ctx->provides.all.data);
698 free(ctx->requires_map);
699 free(ctx->provides_map);
701 fprintf(stderr, "parsed %d requires, %d unique\n",
702 ctx->requires.all.size / sizeof(struct import_property),
703 ctx->set->requires.size / sizeof(struct razor_property));
704 fprintf(stderr, "parsed %d provides, %d unique\n",
705 ctx->provides.all.size / sizeof(struct import_property),
706 ctx->set->provides.size / sizeof(struct razor_property));
711 /* Import a yum filelist as a razor package set. */
715 YUM_STATE_PACKAGE_NAME
719 struct import_context ctx;
720 struct import_property_context *current_property_context;
726 yum_start_element(void *data, const char *name, const char **atts)
728 struct yum_context *ctx = data;
729 const char *n, *version;
732 if (strcmp(name, "name") == 0) {
733 ctx->state = YUM_STATE_PACKAGE_NAME;
734 } else if (strcmp(name, "version") == 0) {
735 for (i = 0; atts[i]; i += 2) {
736 if (strcmp(atts[i], "ver") == 0)
737 version = atts[i + 1];
739 import_context_add_package(&ctx->ctx, ctx->name, version);
740 } else if (strcmp(name, "rpm:requires") == 0) {
741 ctx->current_property_context = &ctx->ctx.requires;
742 } else if (strcmp(name, "rpm:provides") == 0) {
743 ctx->current_property_context = &ctx->ctx.provides;
744 } else if (strcmp(name, "rpm:entry") == 0 &&
745 ctx->current_property_context != NULL) {
748 for (i = 0; atts[i]; i += 2) {
749 if (strcmp(atts[i], "name") == 0)
751 else if (strcmp(atts[i], "ver") == 0)
752 version = atts[i + 1];
756 fprintf(stderr, "invalid rpm:entry, "
757 "missing name or version attributes\n");
761 import_context_add_property(&ctx->ctx,
762 ctx->current_property_context,
768 yum_end_element (void *data, const char *name)
770 struct yum_context *ctx = data;
772 if (strcmp(name, "package") == 0) {
774 import_context_finish_package(&ctx->ctx);
775 } else if (strcmp(name, "name") == 0) {
777 } else if (strcmp(name, "rpm:requires") == 0) {
778 ctx->current_property_context = NULL;
779 } else if (strcmp(name, "rpm:provides") == 0) {
780 ctx->current_property_context = NULL;
785 yum_character_data (void *data, const XML_Char *s, int len)
787 struct yum_context *ctx = data;
789 if (ctx->state == YUM_STATE_PACKAGE_NAME)
790 ctx->name = strndup(s, len);
793 static struct razor_set *
794 razor_set_create_from_yum_filelist(int fd)
796 struct yum_context ctx;
801 razor_prepare_import(&ctx.ctx);
803 parser = XML_ParserCreate(NULL);
804 XML_SetUserData(parser, &ctx);
805 XML_SetElementHandler(parser, yum_start_element, yum_end_element);
806 XML_SetCharacterDataHandler(parser, yum_character_data);
809 len = read(fd, buf, sizeof buf);
812 "couldn't read input: %s\n", strerror(errno));
817 if (XML_Parse(parser, buf, len, 0) == XML_STATUS_ERROR) {
820 XML_ErrorString(XML_GetErrorCode(parser)),
821 XML_GetCurrentLineNumber(parser));
826 XML_ParserFree(parser);
828 return razor_finish_import(&ctx.ctx);
832 razor_set_list(struct razor_set *set)
834 struct razor_package *p, *end;
837 pool = set->string_pool.data;
838 end = set->packages.data + set->packages.size;
839 for (p = set->packages.data; p < end; p++)
840 printf("%s %s\n", &pool[p->name], &pool[p->version]);
843 struct razor_set *bsearch_set;
846 compare_package_name(const void *key, const void *data)
848 const struct razor_package *p = data;
851 pool = bsearch_set->string_pool.data;
853 return strcmp(key, &pool[p->name]);
856 struct razor_package *
857 razor_set_get_package(struct razor_set *set, const char *package)
860 return bsearch(package, set->packages.data,
861 set->packages.size / sizeof(struct razor_package),
862 sizeof(struct razor_package), compare_package_name);
866 razor_set_list_all_properties(struct razor_set *set, struct array *properties)
868 struct razor_property *p, *end;
871 pool = set->string_pool.data;
872 end = properties->data + properties->size;
873 for (p = properties->data; p < end; p++)
874 printf("%s %s\n", &pool[p->name], &pool[p->version]);
878 razor_set_list_requires(struct razor_set *set, const char *name)
880 struct razor_property *p, *requires;
881 struct razor_package *package;
886 package = razor_set_get_package(set, name);
887 r = (unsigned long *) set->property_pool.data +
889 requires = set->requires.data;
890 pool = set->string_pool.data;
893 printf("%s %s\n", &pool[p->name], &pool[p->version]);
896 razor_set_list_all_properties(set, &set->requires);
900 razor_set_list_provides(struct razor_set *set, const char *name)
902 struct razor_property *p, *provides;
903 struct razor_package *package;
908 package = razor_set_get_package(set, name);
909 r = (unsigned long *) set->property_pool.data +
911 provides = set->provides.data;
912 pool = set->string_pool.data;
915 printf("%s %s\n", &pool[p->name], &pool[p->version]);
918 razor_set_list_all_properties(set, &set->provides);
922 razor_set_info(struct razor_set *set)
924 unsigned int offset, size;
927 for (i = 0; i < set->header->sections[i].type; i++) {
928 offset = set->header->sections[i].offset;
929 size = set->header->sections[i + 1].offset - offset;
931 switch (set->header->sections[i].type) {
933 printf("string pool:\t\t%dkb\n", size / 1024);
936 printf("package section:\t%dkb\n", size / 1024);
939 printf("requires section:\t%dkb\n", size / 1024);
942 printf("provides section:\t%dkb\n", size / 1024);
951 printf("usage: razor [ import FILES | lookup <key> | "
952 "list | list-requires | list-provides | eat-yum | info ]\n");
956 static const char *repo_filename = "system.repo";
957 static const char rawhide_repo_filename[] = "rawhide.repo";
960 main(int argc, char *argv[])
963 struct razor_set *set;
965 struct import_context ctx;
968 repo = getenv("RAZOR_REPO");
970 repo_filename = repo;
974 } else if (strcmp(argv[1], "import") == 0) {
975 if (stat("set", &statbuf) && mkdir("set", 0777)) {
976 fprintf(stderr, "could not create directory 'set'\n");
980 razor_prepare_import(&ctx);
982 for (i = 2; i < argc; i++) {
983 if (razor_import(&ctx, argv[i]) < 0) {
984 fprintf(stderr, "failed to import %s\n",
990 set = razor_finish_import(&ctx);
992 printf("bucket allocation: %d\n", set->buckets.alloc);
993 printf("pool size: %d\n", set->string_pool.size);
994 printf("pool allocation: %d\n", set->string_pool.alloc);
995 printf("packages: %d\n",
996 set->packages.size / sizeof(struct razor_package));
997 printf("requires: %d\n",
998 set->requires.size / sizeof(struct razor_property));
999 printf("provides: %d\n",
1000 set->provides.size / sizeof(struct razor_property));
1002 razor_set_write(set, repo_filename);
1004 razor_set_destroy(set);
1005 } else if (strcmp(argv[1], "lookup") == 0) {
1006 set = razor_set_open(repo_filename);
1007 printf("%s is %lu\n", argv[2],
1008 razor_set_lookup(set, argv[2]));
1009 razor_set_destroy(set);
1010 } else if (strcmp(argv[1], "list") == 0) {
1011 set = razor_set_open(repo_filename);
1012 razor_set_list(set);
1013 razor_set_destroy(set);
1014 } else if (strcmp(argv[1], "list-requires") == 0) {
1015 set = razor_set_open(repo_filename);
1016 razor_set_list_requires(set, argv[2]);
1017 razor_set_destroy(set);
1018 } else if (strcmp(argv[1], "list-provides") == 0) {
1019 set = razor_set_open(repo_filename);
1020 razor_set_list_provides(set, argv[2]);
1021 razor_set_destroy(set);
1022 } else if (strcmp(argv[1], "info") == 0) {
1023 set = razor_set_open(repo_filename);
1024 razor_set_info(set);
1025 razor_set_destroy(set);
1026 } else if (strcmp(argv[1], "eat-yum") == 0) {
1027 set = razor_set_create_from_yum_filelist(STDIN_FILENO);
1030 razor_set_write(set, rawhide_repo_filename);
1031 razor_set_destroy(set);