Refactor the import interface and make it opaque.
16 #define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
23 struct razor_set_section {
29 struct razor_set_header {
32 struct razor_set_section sections[0];
35 #define RAZOR_MAGIC 0x7a7a7a7a
36 #define RAZOR_VERSION 1
38 #define RAZOR_PACKAGES 0
39 #define RAZOR_REQUIRES 1
40 #define RAZOR_PROVIDES 2
41 #define RAZOR_STRING_POOL 3
42 #define RAZOR_PROPERTY_POOL 4
44 struct razor_package {
46 unsigned long version;
47 unsigned long requires;
48 unsigned long provides;
51 struct razor_property {
53 unsigned long version;
54 unsigned long packages;
59 struct array string_pool;
60 struct array property_pool;
61 struct array packages;
62 struct array requires;
63 struct array provides;
64 struct razor_set_header *header;
67 struct import_property_context {
72 struct razor_importer {
73 struct razor_set *set;
74 struct import_property_context requires;
75 struct import_property_context provides;
76 struct razor_package *package;
77 unsigned long *requires_map;
78 unsigned long *provides_map;
82 array_init(struct array *array)
84 memset(array, 0, sizeof *array);
88 array_release(struct array *array)
94 array_add(struct array *array, int size)
100 alloc = array->alloc;
104 while (alloc < array->size + size)
107 if (array->alloc < alloc) {
108 data = realloc(array->data, alloc);
112 array->alloc = alloc;
115 p = array->data + array->size;
122 write_to_fd(int fd, void *p, size_t size)
128 len = write(fd, p, rest);
148 struct razor_set_section razor_sections[] = {
149 { RAZOR_PACKAGES, offsetof(struct razor_set, packages) },
150 { RAZOR_REQUIRES, offsetof(struct razor_set, requires) },
151 { RAZOR_PROVIDES, offsetof(struct razor_set, provides) },
152 { RAZOR_STRING_POOL, offsetof(struct razor_set, string_pool) },
153 { RAZOR_PROPERTY_POOL, offsetof(struct razor_set, property_pool) },
157 razor_set_create(void)
159 return zalloc(sizeof(struct razor_set));
163 razor_set_open(const char *filename)
165 struct razor_set *set;
166 struct razor_set_section *s;
171 set = zalloc(sizeof *set);
172 fd = open(filename, O_RDONLY);
173 if (fstat(fd, &stat) < 0)
175 set->header = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
176 if (set->header == MAP_FAILED) {
181 for (s = set->header->sections; ~s->type; s++) {
182 if (s->type >= ARRAY_SIZE(razor_sections))
184 if (s->type != razor_sections[s->type].type)
186 array = (void *) set + razor_sections[s->type].offset;
187 array->data = (void *) set->header + s->offset;
188 array->size = s->size;
189 array->alloc = s->size;
197 razor_set_destroy(struct razor_set *set)
204 for (i = 0; set->header->sections[i].type; i++)
206 size = set->header->sections[i].type;
207 munmap(set->header, size);
208 free(set->buckets.data);
210 for (i = 0; i < ARRAY_SIZE(razor_sections); i++) {
211 a = (void *) set + razor_sections[i].offset;
214 free(set->buckets.data);
221 razor_set_write(struct razor_set *set, const char *filename)
224 struct razor_set_header *header = (struct razor_set_header *) data;
226 unsigned long offset;
229 memset(data, 0, sizeof data);
230 header->magic = RAZOR_MAGIC;
231 header->version = RAZOR_VERSION;
232 offset = sizeof data;
234 for (i = 0; i < ARRAY_SIZE(razor_sections); i++) {
235 if (razor_sections[i].type != i)
237 a = (void *) set + razor_sections[i].offset;
238 header->sections[i].type = i;
239 header->sections[i].offset = offset;
240 header->sections[i].size = a->size;
241 offset += (a->size + 4095) & ~4095;
244 header->sections[i].type = ~0;
245 header->sections[i].offset = 0;
246 header->sections[i].size = 0;
248 fd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0666);
252 write_to_fd(fd, data, sizeof data);
253 for (i = 0; i < ARRAY_SIZE(razor_sections); i++) {
254 if (razor_sections[i].type != i)
256 a = (void *) set + razor_sections[i].offset;
257 write_to_fd(fd, a->data, (a->size + 4095) & ~4095);
266 hash_string(const char *key)
269 unsigned int hash = 0;
271 for (p = key; *p; p++)
272 hash = (hash * 617) ^ *p;
278 razor_set_lookup(struct razor_set *set, const char *key)
280 unsigned int mask, start, i;
284 pool = set->string_pool.data;
285 mask = set->buckets.alloc - 1;
286 start = hash_string(key) * sizeof(unsigned long);
288 for (i = 0; i < set->buckets.alloc; i += sizeof *b) {
289 b = set->buckets.data + ((start + i) & mask);
294 if (strcmp(key, &pool[*b]) == 0)
302 add_to_string_pool(struct razor_set *set, const char *key)
307 len = strlen(key) + 1;
308 p = array_add(&set->string_pool, len);
311 return p - (char *) set->string_pool.data;
315 add_to_property_pool(struct razor_set *set, struct array *properties)
319 p = array_add(properties, sizeof *p);
321 p = array_add(&set->property_pool, properties->size);
322 memcpy(p, properties->data, properties->size);
324 return p - (unsigned long *) set->property_pool.data;
328 do_insert(struct razor_set *set, unsigned long value)
330 unsigned int mask, start, i;
334 key = (char *) set->string_pool.data + value;
335 mask = set->buckets.alloc - 1;
336 start = hash_string(key) * sizeof(unsigned long);
338 for (i = 0; i < set->buckets.alloc; i += sizeof *b) {
339 b = set->buckets.data + ((start + i) & mask);
348 razor_set_insert(struct razor_set *set, const char *key)
350 unsigned long value, *buckets, *b, *end;
353 alloc = set->buckets.alloc;
354 array_add(&set->buckets, 4 * sizeof *buckets);
355 if (alloc != set->buckets.alloc) {
356 end = set->buckets.data + alloc;
357 memset(end, 0, set->buckets.alloc - alloc);
358 for (b = set->buckets.data; b < end; b++) {
362 do_insert(set, value);
367 value = add_to_string_pool(set, key);
368 do_insert (set, value);
374 razor_set_tokenize(struct razor_set *set, const char *string)
379 return razor_set_tokenize(set, "");
381 token = razor_set_lookup(set, string);
385 return razor_set_insert(set, string);
389 razor_importer_begin_package(struct razor_importer *importer,
390 const char *name, const char *version)
392 struct razor_package *p;
394 p = array_add(&importer->set->packages, sizeof *p);
395 p->name = razor_set_tokenize(importer->set, name);
396 p->version = razor_set_tokenize(importer->set, version);
398 importer->package = p;
399 array_init(&importer->requires.package);
400 array_init(&importer->provides.package);
404 razor_importer_finish_package(struct razor_importer *importer)
406 struct razor_package *p;
408 p = importer->package;
409 p->requires = add_to_property_pool(importer->set,
410 &importer->requires.package);
411 p->provides = add_to_property_pool(importer->set,
412 &importer->provides.package);
414 array_release(&importer->requires.package);
415 array_release(&importer->provides.package);
419 razor_importer_add_property(struct razor_importer *importer,
420 struct import_property_context *pctx,
421 const char *name, const char *version)
423 struct razor_property *p;
426 p = array_add(pctx->all, sizeof *p);
427 p->name = razor_set_tokenize(importer->set, name);
428 p->version = razor_set_tokenize(importer->set, version);
429 p->packages = importer->package -
430 (struct razor_package *) importer->set->packages.data;
432 r = array_add(&pctx->package, sizeof *r);
433 *r = p - (struct razor_property *) pctx->all->data;
437 razor_importer_add_requires(struct razor_importer *importer,
438 const char *name, const char *version)
440 razor_importer_add_property(importer,
441 &importer->requires, name, version);
445 razor_importer_add_provides(struct razor_importer *importer,
446 const char *name, const char *version)
448 razor_importer_add_property(importer,
449 &importer->provides, name, version);
452 struct razor_importer *
453 razor_importer_new(void)
455 struct razor_importer *importer;
457 importer = zalloc(sizeof *importer);
458 importer->set = razor_set_create();
459 importer->requires.all = &importer->set->requires;
460 importer->provides.all = &importer->set->provides;
465 typedef int (*compare_with_data_func_t)(const void *p1,
469 struct qsort_context {
471 compare_with_data_func_t compare;
476 qsort_swap(void *p1, void *p2, size_t size)
480 memcpy(buffer, p1, size);
481 memcpy(p1, p2, size);
482 memcpy(p2, buffer, size);
486 __qsort_with_data(void *base, size_t nelem, unsigned long *map,
487 struct qsort_context *ctx)
489 void *p, *start, *end, *pivot;
490 unsigned long *mp, *mstart, *mend, tmp;
491 int left, right, result;
492 size_t size = ctx->size;
496 end = base + nelem * size;
500 pivot = base + (random() % nelem) * size;
503 result = ctx->compare(p, pivot, ctx->data);
505 qsort_swap(p, start, size);
515 } else if (result == 0) {
521 qsort_swap(p, end, size);
530 left = (start - base) / size;
531 right = (base + nelem * size - end) / size;
533 __qsort_with_data(base, left, map, ctx);
535 __qsort_with_data(end, right, mend, ctx);
539 qsort_with_data(void *base, size_t nelem, size_t size,
540 compare_with_data_func_t compare, void *data)
542 struct qsort_context ctx;
547 ctx.compare = compare;
550 map = malloc(nelem * sizeof (unsigned long));
551 for (i = 0; i < nelem; i++)
554 __qsort_with_data(base, nelem, map, &ctx);
560 compare_packages(const void *p1, const void *p2, void *data)
562 const struct razor_package *pkg1 = p1, *pkg2 = p2;
563 struct razor_set *set = data;
564 char *pool = set->string_pool.data;
566 if (pkg1->name == pkg2->name)
569 return strcmp(&pool[pkg1->name], &pool[pkg2->name]);
573 compare_properties(const void *p1, const void *p2, void *data)
575 const struct razor_property *prop1 = p1, *prop2 = p2;
576 struct razor_set *set = data;
577 char *pool = set->string_pool.data;
579 if (prop1->name == prop2->name)
580 return strcmp(&pool[prop1->version], &pool[prop2->version]);
582 return strcmp(&pool[prop1->name], &pool[prop2->name]);
585 static unsigned long *
586 uniqueify_properties(struct razor_set *set, struct array *properties)
588 struct razor_property *rp, *up, *rp_end;
589 struct array *pkgs, *p;
590 unsigned long *map, *rmap, *r;
591 int i, count, unique;
593 count = properties->size / sizeof(struct razor_property);
594 map = qsort_with_data(properties->data,
596 sizeof(struct razor_property),
600 rp_end = properties->data + properties->size;
601 rmap = malloc(count * sizeof *map);
602 pkgs = zalloc(count * sizeof *pkgs);
603 for (rp = properties->data, up = rp, i = 0; rp < rp_end; rp++, i++) {
604 if (rp->name != up->name || rp->version != up->version) {
607 up->version = rp->version;
610 unique = up - (struct razor_property *) properties->data;
611 rmap[map[i]] = unique;
612 r = array_add(&pkgs[unique], sizeof *r);
618 properties->size = (void *) up - properties->data;
620 for (rp = properties->data, p = pkgs; rp < rp_end; rp++, p++) {
621 rp->packages = add_to_property_pool(set, p);
631 remap_package_links(struct razor_importer *importer)
633 struct razor_package *p, *end;
634 unsigned long *pool, *r;
636 pool = importer->set->property_pool.data;
637 end = importer->set->packages.data + importer->set->packages.size;
638 for (p = importer->set->packages.data; p < end; p++) {
639 for (r = &pool[p->requires]; ~*r; r++)
640 *r = importer->requires_map[*r];
641 for (r = &pool[p->provides]; ~*r; r++)
642 *r = importer->provides_map[*r];
647 remap_property_links(struct razor_importer *importer, unsigned long *map)
649 struct razor_property *p, *end;
650 struct razor_package *rp;
651 unsigned long *pool, *r, *rmap;
654 pool = importer->set->property_pool.data;
655 count = importer->set->packages.size / sizeof(struct razor_package);
656 rmap = malloc(count * sizeof *map);
657 rp = importer->set->packages.data;
658 for (i = 0; i < count; i++)
661 /* FIXME: This will break if we implement package list sharing
662 * for all properties, since we'll remap those lists more than
663 * once. We should just have a separate pool for property
664 * lists and a separate pool for package lists and remap it as
665 * a flat pool. Right now, as property lists and package
666 * lists are mixed, we can't do that. */
668 end = importer->set->requires.data + importer->set->requires.size;
669 for (p = importer->set->requires.data; p < end; p++)
670 for (r = &pool[p->packages]; ~*r; r++)
673 end = importer->set->provides.data + importer->set->provides.size;
674 for (p = importer->set->provides.data; p < end; p++)
675 for (r = &pool[p->packages]; ~*r; r++)
682 razor_importer_finish(struct razor_importer *importer)
684 struct razor_set *set;
688 importer->requires_map = uniqueify_properties(importer->set,
689 importer->requires.all);
690 importer->provides_map = uniqueify_properties(importer->set,
691 importer->provides.all);
692 remap_package_links(importer);
693 free(importer->requires_map);
694 free(importer->provides_map);
696 count = importer->set->packages.size / sizeof(struct razor_package);
697 map = qsort_with_data(importer->set->packages.data,
699 sizeof(struct razor_package),
702 remap_property_links(importer, map);
712 razor_set_list(struct razor_set *set)
714 struct razor_package *p, *end;
717 pool = set->string_pool.data;
718 end = set->packages.data + set->packages.size;
719 for (p = set->packages.data; p < end; p++)
720 printf("%s %s\n", &pool[p->name], &pool[p->version]);
723 struct razor_set *bsearch_set;
726 compare_package_name(const void *key, const void *data)
728 const struct razor_package *p = data;
731 pool = bsearch_set->string_pool.data;
733 return strcmp(key, &pool[p->name]);
736 struct razor_package *
737 razor_set_get_package(struct razor_set *set, const char *package)
740 return bsearch(package, set->packages.data,
741 set->packages.size / sizeof(struct razor_package),
742 sizeof(struct razor_package), compare_package_name);
746 compare_property_name(const void *key, const void *data)
748 const struct razor_property *p = data;
751 pool = bsearch_set->string_pool.data;
753 return strcmp(key, &pool[p->name]);
756 struct razor_property *
757 razor_set_get_property(struct razor_set *set,
758 struct array *properties,
759 const char *property)
761 struct razor_property *p, *start;
764 p = bsearch(property, properties->data,
765 properties->size / sizeof(struct razor_property),
766 sizeof(struct razor_property), compare_property_name);
768 start = properties->data;
769 while (p > start && (p - 1)->name == p->name)
776 razor_set_list_all_properties(struct razor_set *set, struct array *properties)
778 struct razor_property *p, *end;
781 pool = set->string_pool.data;
782 end = properties->data + properties->size;
783 for (p = properties->data; p < end; p++)
784 printf("%s %s\n", &pool[p->name], &pool[p->version]);
788 razor_set_list_requires(struct razor_set *set, const char *name)
790 struct razor_property *p, *requires;
791 struct razor_package *package;
796 package = razor_set_get_package(set, name);
797 r = (unsigned long *) set->property_pool.data +
799 requires = set->requires.data;
800 pool = set->string_pool.data;
803 printf("%s %s\n", &pool[p->name], &pool[p->version]);
806 razor_set_list_all_properties(set, &set->requires);
810 razor_set_list_provides(struct razor_set *set, const char *name)
812 struct razor_property *p, *provides;
813 struct razor_package *package;
818 package = razor_set_get_package(set, name);
819 r = (unsigned long *) set->property_pool.data +
821 provides = set->provides.data;
822 pool = set->string_pool.data;
825 printf("%s %s\n", &pool[p->name], &pool[p->version]);
828 razor_set_list_all_properties(set, &set->provides);
832 razor_set_list_property_packages(struct razor_set *set,
833 struct array *properties,
837 struct razor_property *property, *end;
838 struct razor_package *p, *packages;
845 property = razor_set_get_property(set, properties, name);
846 packages = set->packages.data;
847 pool = set->string_pool.data;
848 end = properties->data + properties->size;
849 while (property < end && strcmp(name, &pool[property->name]) == 0) {
850 if (version && strcmp(version, &pool[property->version]) != 0)
852 r = (unsigned long *)
853 set->property_pool.data + property->packages;
857 &pool[p->name], &pool[p->version]);
865 razor_set_validate(struct razor_set *set, struct array *unsatisfied)
867 struct razor_property *r, *p, *rend, *pend;
871 r = set->requires.data;
872 p = set->provides.data;
873 rend = set->requires.data + set->requires.size;
874 pend = set->provides.data + set->provides.size;
875 pool = set->string_pool.data;
878 while (p < pend && strcmp(&pool[r->name], &pool[p->name]) > 0)
880 if (p == pend || strcmp(&pool[r->name], &pool[p->name]) != 0) {
881 u = array_add(unsatisfied, sizeof *u);
882 *u = r - (struct razor_property *) set->requires.data;
889 razor_set_list_unsatisfied(struct razor_set *set)
891 struct array unsatisfied;
892 struct razor_property *requires, *r;
893 unsigned long *u, *end;
896 array_init(&unsatisfied);
897 razor_set_validate(set, &unsatisfied);
899 end = unsatisfied.data + unsatisfied.size;
900 requires = set->requires.data;
901 pool = set->string_pool.data;
903 for (u = unsatisfied.data; u < end; u++) {
905 printf("%s %s not satisfied\n",
906 &pool[r->name], &pool[r->version]);
909 array_release(&unsatisfied);
913 razor_set_info(struct razor_set *set)
915 unsigned int offset, size;
918 for (i = 0; i < set->header->sections[i].type; i++) {
919 offset = set->header->sections[i].offset;
920 size = set->header->sections[i].size;
922 switch (set->header->sections[i].type) {
924 printf("package section:\t%dkb\n", size / 1024);
927 printf("requires section:\t%dkb\n", size / 1024);
930 printf("provides section:\t%dkb\n", size / 1024);
932 case RAZOR_STRING_POOL:
933 printf("string pool:\t\t%dkb\n", size / 1024);
935 case RAZOR_PROPERTY_POOL:
936 printf("properties section:\t%dkb\n", size / 1024);
945 printf("usage: razor [ import FILES | lookup <key> | "
946 "list | list-requires | list-provides | eat-yum | info ]\n");
950 static const char *repo_filename = "system.repo";
951 static const char rawhide_repo_filename[] = "rawhide.repo";
954 main(int argc, const char *argv[])
956 struct razor_set *set;
960 repo = getenv("RAZOR_REPO");
962 repo_filename = repo;
966 } else if (strcmp(argv[1], "import") == 0) {
967 if (stat("set", &statbuf) && mkdir("set", 0777)) {
968 fprintf(stderr, "could not create directory 'set'\n");
972 set = razor_import_rzr_files(argc - 2, argv + 2);
974 printf("bucket allocation: %d\n", set->buckets.alloc);
975 printf("pool size: %d\n", set->string_pool.size);
976 printf("pool allocation: %d\n", set->string_pool.alloc);
977 printf("packages: %d\n",
978 set->packages.size / sizeof(struct razor_package));
979 printf("requires: %d\n",
980 set->requires.size / sizeof(struct razor_property));
981 printf("provides: %d\n",
982 set->provides.size / sizeof(struct razor_property));
984 razor_set_write(set, repo_filename);
986 razor_set_destroy(set);
987 } else if (strcmp(argv[1], "lookup") == 0) {
988 set = razor_set_open(repo_filename);
989 printf("%s is %lu\n", argv[2],
990 razor_set_lookup(set, argv[2]));
991 razor_set_destroy(set);
992 } else if (strcmp(argv[1], "list") == 0) {
993 set = razor_set_open(repo_filename);
995 razor_set_destroy(set);
996 } else if (strcmp(argv[1], "list-requires") == 0) {
997 set = razor_set_open(repo_filename);
998 razor_set_list_requires(set, argv[2]);
999 razor_set_destroy(set);
1000 } else if (strcmp(argv[1], "list-provides") == 0) {
1001 set = razor_set_open(repo_filename);
1002 razor_set_list_provides(set, argv[2]);
1003 razor_set_destroy(set);
1004 } else if (strcmp(argv[1], "what-requires") == 0) {
1005 set = razor_set_open(repo_filename);
1006 razor_set_list_property_packages(set, &set->requires,
1008 razor_set_destroy(set);
1009 } else if (strcmp(argv[1], "what-provides") == 0) {
1010 set = razor_set_open(repo_filename);
1011 razor_set_list_property_packages(set, &set->provides,
1013 razor_set_destroy(set);
1014 } else if (strcmp(argv[1], "info") == 0) {
1015 set = razor_set_open(repo_filename);
1016 razor_set_info(set);
1017 razor_set_destroy(set);
1018 } else if (strcmp(argv[1], "eat-yum") == 0) {
1019 set = razor_set_create_from_yum_filelist(STDIN_FILENO);
1022 razor_set_write(set, rawhide_repo_filename);
1023 razor_set_destroy(set);
1024 printf("wrote %s\n", rawhide_repo_filename);
1025 } else if (strcmp(argv[1], "import-rpmdb") == 0) {
1026 set = razor_set_create_from_rpmdb();
1029 razor_set_write(set, repo_filename);
1030 razor_set_destroy(set);
1031 printf("wrote %s\n", repo_filename);
1032 } else if (strcmp(argv[1], "validate") == 0) {
1033 set = razor_set_open(repo_filename);
1036 razor_set_list_unsatisfied(set);
1037 razor_set_destroy(set);