Clean up import logic and make it independent of XML parsing.
19 array_init(struct array *array)
21 memset(array, 0, sizeof *array);
25 array_release(struct array *array)
31 array_add(struct array *array, int size)
41 while (alloc < array->size + size)
44 if (array->alloc < alloc) {
45 data = realloc(array->data, alloc);
52 p = array->data + array->size;
59 write_to_fd(int fd, void *p, size_t size)
65 len = write(fd, p, rest);
75 write_to_file(const char *filename, void *p, size_t size)
79 fd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0666);
82 err = write_to_fd(fd, p, size);
99 struct razor_set_header {
101 unsigned int version;
102 struct { unsigned int type, offset; } sections[0];
105 #define RAZOR_MAGIC 0x7a7a7a7a
106 #define RAZOR_VERSION 1
108 #define RAZOR_BUCKETS 1
109 #define RAZOR_STRINGS 2
110 #define RAZOR_PACKAGES 3
111 #define RAZOR_REQUIRES 4
112 #define RAZOR_PROVIDES 5
113 #define RAZOR_PROPERTIES 6
115 struct razor_package {
117 unsigned long version;
118 unsigned long requires;
119 unsigned long provides;
122 struct razor_property {
124 unsigned long version;
125 unsigned long packages;
129 struct array buckets;
130 struct array string_pool;
131 struct array property_pool;
132 struct array packages;
133 struct array requires;
134 struct array provides;
135 struct razor_set_header *header;
139 razor_set_create(void)
141 struct razor_set *set;
144 set = zalloc(sizeof(struct razor_set));
145 p = array_add(&set->string_pool, 1);
152 razor_set_open(const char *filename)
154 struct razor_set *set;
156 unsigned int size, offset;
159 set = zalloc(sizeof *set);
160 fd = open(filename, O_RDONLY);
161 if (fstat(fd, &stat) < 0)
163 set->header = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
164 if (set->header == MAP_FAILED) {
169 for (i = 0; i < set->header->sections[i].type; i++) {
170 offset = set->header->sections[i].offset;
171 size = set->header->sections[i + 1].offset - offset;
173 switch (set->header->sections[i].type) {
175 set->buckets.data = (void *) set->header + offset;
176 set->buckets.size = size;
177 set->buckets.alloc = size;
180 set->string_pool.data = (void *) set->header + offset;
181 set->string_pool.size = size;
182 set->string_pool.alloc = size;
185 set->packages.data = (void *) set->header + offset;
186 set->packages.size = size;
187 set->packages.size = size;
190 set->requires.data = (void *) set->header + offset;
191 set->requires.size = size;
192 set->requires.size = size;
195 set->provides.data = (void *) set->header + offset;
196 set->provides.size = size;
197 set->provides.size = size;
199 case RAZOR_PROPERTIES:
200 set->property_pool.data = (void *) set->header + offset;
201 set->property_pool.size = size;
202 set->property_pool.size = size;
212 razor_set_destroy(struct razor_set *set)
218 for (i = 0; set->header->sections[i].type; i++)
220 size = set->header->sections[i].type;
221 munmap(set->header, size);
223 free(set->buckets.data);
224 free(set->string_pool.data);
225 free(set->packages.data);
226 free(set->requires.data);
227 free(set->provides.data);
228 free(set->property_pool.data);
235 razor_set_write(struct razor_set *set, const char *filename)
238 struct razor_set_header *header = (struct razor_set_header *) data;
239 int fd, pool_size, packages_size, requires_size, provides_size;
242 /* Align these to pages sizes */
243 pool_size = (set->string_pool.size + 4095) & ~4095;
244 packages_size = (set->packages.size + 4095) & ~4095;
245 requires_size = (set->requires.size + 4095) & ~4095;
246 provides_size = (set->provides.size + 4095) & ~4095;
247 properties_size = (set->property_pool.size + 4095) & ~4095;
249 memset(data, 0, sizeof data);
250 header->magic = RAZOR_MAGIC;
251 header->version = RAZOR_VERSION;
253 header->sections[0].type = RAZOR_BUCKETS;
254 header->sections[0].offset = sizeof data;
256 header->sections[1].type = RAZOR_STRINGS;
257 header->sections[1].offset =
258 header->sections[0].offset + set->buckets.alloc;
260 header->sections[2].type = RAZOR_PACKAGES;
261 header->sections[2].offset =
262 header->sections[1].offset + pool_size;
264 header->sections[3].type = RAZOR_REQUIRES;
265 header->sections[3].offset =
266 header->sections[2].offset + packages_size;
268 header->sections[4].type = RAZOR_PROVIDES;
269 header->sections[4].offset =
270 header->sections[3].offset + requires_size;
272 header->sections[5].type = RAZOR_PROPERTIES;
273 header->sections[5].offset =
274 header->sections[4].offset + provides_size;
276 header->sections[6].type = 0;
277 header->sections[6].offset =
278 header->sections[5].offset + properties_size;
280 fd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0666);
284 write_to_fd(fd, data, sizeof data);
285 write_to_fd(fd, set->buckets.data, set->buckets.alloc);
286 write_to_fd(fd, set->string_pool.data, pool_size);
287 write_to_fd(fd, set->packages.data, packages_size);
288 write_to_fd(fd, set->requires.data, requires_size);
289 write_to_fd(fd, set->provides.data, provides_size);
290 write_to_fd(fd, set->property_pool.data, properties_size);
296 hash_string(const char *key)
299 unsigned int hash = 0;
301 for (p = key; *p; p++)
302 hash = (hash * 617) ^ *p;
308 razor_set_lookup(struct razor_set *set, const char *key)
310 unsigned int mask, start, i;
314 pool = set->string_pool.data;
315 mask = set->buckets.alloc - 1;
316 start = hash_string(key) * sizeof(unsigned long);
318 for (i = 0; i < set->buckets.alloc; i += sizeof *b) {
319 b = set->buckets.data + ((start + i) & mask);
324 if (strcmp(key, &pool[*b]) == 0)
332 add_to_string_pool(struct razor_set *set, const char *key)
337 len = strlen(key) + 1;
338 p = array_add(&set->string_pool, len);
341 return p - (char *) set->string_pool.data;
345 add_to_property_pool(struct razor_set *set, struct array *properties)
349 p = array_add(properties, sizeof *p);
351 p = array_add(&set->property_pool, properties->size);
352 memcpy(p, properties->data, properties->size);
354 return p - (unsigned long *) set->property_pool.data;
358 do_insert(struct razor_set *set, unsigned long value)
360 unsigned int mask, start, i;
364 key = (char *) set->string_pool.data + value;
365 mask = set->buckets.alloc - 1;
366 start = hash_string(key) * sizeof(unsigned long);
368 for (i = 0; i < set->buckets.alloc; i += sizeof *b) {
369 b = set->buckets.data + ((start + i) & mask);
378 razor_set_insert(struct razor_set *set, const char *key)
380 unsigned long value, *buckets, *b, *end;
383 alloc = set->buckets.alloc;
384 array_add(&set->buckets, 4 * sizeof *buckets);
385 if (alloc != set->buckets.alloc) {
386 end = set->buckets.data + alloc;
387 memset(end, 0, set->buckets.alloc - alloc);
388 for (b = set->buckets.data; b < end; b++) {
392 do_insert(set, value);
397 value = add_to_string_pool(set, key);
398 do_insert (set, value);
404 razor_set_tokenize(struct razor_set *set, const char *string)
411 token = razor_set_lookup(set, string);
415 return razor_set_insert(set, string);
418 struct import_property_context {
420 struct array package;
423 struct import_context {
424 struct razor_set *set;
425 struct import_property_context requires;
426 struct import_property_context provides;
427 unsigned long package;
428 unsigned long *requires_map;
429 unsigned long *provides_map;
432 struct import_property {
434 unsigned long version;
435 unsigned long package;
437 unsigned long unique_index;
441 import_context_add_package(struct import_context *ctx,
442 const char *name, const char *version)
444 struct razor_package *p;
446 p = array_add(&ctx->set->packages, sizeof *p);
447 p->name = razor_set_tokenize(ctx->set, name);
448 p->version = razor_set_tokenize(ctx->set, version);
450 ctx->package = p - (struct razor_package *) ctx->set->packages.data;
451 array_init(&ctx->requires.package);
452 array_init(&ctx->provides.package);
456 import_context_finish_package(struct import_context *ctx)
458 struct razor_package *p;
460 p = (struct razor_package *) ctx->set->packages.data + ctx->package;
461 p->requires = add_to_property_pool(ctx->set, &ctx->requires.package);
462 p->provides = add_to_property_pool(ctx->set, &ctx->provides.package);
464 array_release(&ctx->requires.package);
465 array_release(&ctx->provides.package);
469 import_context_add_property(struct import_context *ctx,
470 struct import_property_context *pctx,
471 const char *name, const char *version)
473 struct import_property *p;
476 p = array_add(&pctx->all, sizeof *p);
477 p->name = razor_set_tokenize(ctx->set, name);
478 p->version = razor_set_tokenize(ctx->set, version);
479 p->package = ctx->package;
480 p->index = p - (struct import_property *) pctx->all.data;
482 r = array_add(&pctx->package, sizeof *r);
487 parse_package(struct import_context *ctx, const char **atts, void *data)
489 const char *name = NULL, *version = NULL;
492 for (i = 0; atts[i]; i += 2) {
493 if (strcmp(atts[i], "name") == 0)
495 else if (strcmp(atts[i], "version") == 0)
496 version = atts[i + 1];
499 if (name == NULL || version == NULL) {
500 fprintf(stderr, "invalid package tag, "
501 "missing name or version attributes\n");
505 import_context_add_package(ctx, name, version);
509 parse_property(struct import_context *ctx, const char **atts, void *data)
511 const char *name = NULL, *version = NULL;
514 for (i = 0; atts[i]; i += 2) {
515 if (strcmp(atts[i], "name") == 0)
517 if (strcmp(atts[i], "version") == 0)
518 version = atts[i + 1];
522 fprintf(stderr, "invalid tag, missing name attribute\n");
526 import_context_add_property(ctx, data, name, version);
530 start_element(void *data, const char *name, const char **atts)
532 struct import_context *ctx = data;
534 if (strcmp(name, "package") == 0)
535 parse_package(ctx, atts, NULL);
536 else if (strcmp(name, "requires") == 0)
537 parse_property(ctx, atts, &ctx->requires);
538 else if (strcmp(name, "provides") == 0)
539 parse_property(ctx, atts, &ctx->provides);
543 end_element (void *data, const char *name)
545 struct import_context *ctx = data;
547 if (strcmp(name, "package") == 0)
548 import_context_finish_package(ctx);
552 sha1_to_hex(const unsigned char *sha1)
555 static char hexbuffer[4][50];
556 static const char hex[] = "0123456789abcdef";
557 char *buffer = hexbuffer[3 & ++bufno], *buf = buffer;
560 for (i = 0; i < 20; i++) {
561 unsigned int val = *sha1++;
562 *buf++ = hex[val >> 4];
563 *buf++ = hex[val & 0xf];
571 razor_prepare_import(struct import_context *ctx)
573 memset(ctx, 0, sizeof *ctx);
574 ctx->set = razor_set_create();
578 razor_import(struct import_context *ctx, const char *filename)
586 unsigned char hash[20];
588 fd = open(filename, O_RDONLY);
589 if (fstat(fd, &stat) < 0)
591 p = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
595 parser = XML_ParserCreate(NULL);
596 XML_SetUserData(parser, ctx);
597 XML_SetElementHandler(parser, start_element, end_element);
598 if (XML_Parse(parser, p, stat.st_size, 1) == XML_STATUS_ERROR) {
600 "%s at line %d, %s\n",
601 XML_ErrorString(XML_GetErrorCode(parser)),
602 XML_GetCurrentLineNumber(parser),
607 XML_ParserFree(parser);
610 SHA1_Update(&sha1, p, stat.st_size);
611 SHA1_Final(hash, &sha1);
615 snprintf(buf, sizeof buf, "set/%s", sha1_to_hex(hash));
616 if (write_to_file(buf, p, stat.st_size) < 0)
618 munmap(p, stat.st_size);
623 static struct razor_set *qsort_set;
626 compare_packages(const void *p1, const void *p2)
628 const struct razor_package *pkg1 = p1, *pkg2 = p2;
629 char *pool = qsort_set->string_pool.data;
631 return strcmp(&pool[pkg1->name], &pool[pkg2->name]);
635 compare_properties(const void *p1, const void *p2)
637 const struct import_property *prop1 = p1, *prop2 = p2;
638 char *pool = qsort_set->string_pool.data;
641 result = strcmp(&pool[prop1->name], &pool[prop2->name]);
643 return strcmp(&pool[prop1->version], &pool[prop2->version]);
648 static unsigned long *
649 uniqueify_properties(struct array *in, struct array *out)
651 struct import_property *ip, *end;
652 struct razor_property *rp;
656 count = in->size / sizeof(struct import_property);
657 qsort(in->data, count,
658 sizeof(struct import_property), compare_properties);
661 end = in->data + in->size;
662 for (ip = in->data; ip < end; ip++) {
664 ip->name != rp->name || ip->version != rp->version) {
665 rp = array_add(out, sizeof *rp);
667 rp->version = ip->version;
669 ip->unique_index = rp - (struct razor_property *) out->data;
672 map = malloc(count * sizeof (unsigned long));
674 for (i = 0; i < count; i++)
675 map[ip[i].index] = ip[i].unique_index;
681 sort_packages(struct import_context *ctx)
683 struct razor_package *p, *end;
684 unsigned long *pool, *r;
686 pool = ctx->set->property_pool.data;
687 end = ctx->set->packages.data + ctx->set->packages.size;
688 for (p = ctx->set->packages.data; p < end; p++) {
689 for (r = &pool[p->requires]; *r; r++)
690 *r = ctx->requires_map[*r];
691 for (r = &pool[p->provides]; *r; r++)
692 *r = ctx->provides_map[*r];
695 qsort(ctx->set->packages.data,
696 ctx->set->packages.size / sizeof(struct razor_package),
697 sizeof(struct razor_package), compare_packages);
700 static struct razor_set *
701 razor_finish_import(struct import_context *ctx)
703 struct razor_package *pkg;
704 struct razor_property *prop;
706 qsort_set = ctx->set;
709 uniqueify_properties(&ctx->requires.all, &ctx->set->requires);
711 uniqueify_properties(&ctx->provides.all, &ctx->set->provides);
715 free(ctx->requires.all.data);
716 free(ctx->provides.all.data);
717 free(ctx->requires_map);
718 free(ctx->provides_map);
720 /* FIXME: We add sentinel package/props here, but we should
721 * probably just have a size field in the header section. */
722 pkg = array_add(&ctx->set->packages, sizeof *pkg);
724 prop = array_add(&ctx->set->requires, sizeof *prop);
726 prop = array_add(&ctx->set->provides, sizeof *prop);
729 fprintf(stderr, "parsed %d requires, %d unique\n",
730 ctx->requires.all.size / sizeof(struct import_property),
731 ctx->set->requires.size / sizeof(struct razor_property));
732 fprintf(stderr, "parsed %d provides, %d unique\n",
733 ctx->provides.all.size / sizeof(struct import_property),
734 ctx->set->provides.size / sizeof(struct razor_property));
740 razor_set_list(struct razor_set *set)
742 struct razor_package *p, *end;
745 pool = set->string_pool.data;
746 end = set->packages.data + set->packages.size;
747 for (p = set->packages.data; p < end && p->name; p++)
748 printf("%s %s\n", &pool[p->name], &pool[p->version]);
751 struct razor_package *
752 razor_set_get_package(struct razor_set *set, const char *package)
755 struct razor_package *p, *end;
757 name = razor_set_lookup(set, package);
758 end = set->packages.data + set->packages.size;
759 for (p = set->packages.data; p < end && p->name; p++)
767 razor_set_list_all_properties(struct razor_set *set, struct array *properties)
769 struct razor_property *p, *end;
772 pool = set->string_pool.data;
773 end = properties->data + properties->size;
774 for (p = properties->data; p < end && p->name; p++)
775 printf("%s %s\n", &pool[p->name], &pool[p->version]);
779 razor_set_list_requires(struct razor_set *set, const char *name)
781 struct razor_property *p, *requires;
782 struct razor_package *package;
787 package = razor_set_get_package(set, name);
788 r = (unsigned long *) set->property_pool.data +
790 requires = set->requires.data;
791 pool = set->string_pool.data;
794 printf("%s %s\n", &pool[p->name], &pool[p->version]);
797 razor_set_list_all_properties(set, &set->requires);
801 razor_set_list_provides(struct razor_set *set, const char *name)
803 struct razor_property *p, *provides;
804 struct razor_package *package;
809 package = razor_set_get_package(set, name);
810 r = (unsigned long *) set->property_pool.data +
812 provides = set->provides.data;
813 pool = set->string_pool.data;
816 printf("%s %s\n", &pool[p->name], &pool[p->version]);
819 razor_set_list_all_properties(set, &set->provides);
823 razor_set_info(struct razor_set *set)
825 unsigned int offset, size;
828 for (i = 0; i < set->header->sections[i].type; i++) {
829 offset = set->header->sections[i].offset;
830 size = set->header->sections[i + 1].offset - offset;
832 switch (set->header->sections[i].type) {
834 printf("bucket section:\t\t%dkb\n", size / 1024);
837 printf("string pool:\t\t%dkb\n", size / 1024);
840 printf("package section:\t%dkb\n", size / 1024);
843 printf("requires section:\t%dkb\n", size / 1024);
846 printf("provides section:\t%dkb\n", size / 1024);
855 printf("usage: razor [ import FILES | lookup <key> | "
856 "list | list-requires | list-provides | info ]\n");
860 static const char repo_filename[] = "system.repo";
863 main(int argc, char *argv[])
866 struct razor_set *set;
868 struct import_context ctx;
872 } else if (strcmp(argv[1], "import") == 0) {
873 if (stat("set", &statbuf) && mkdir("set", 0777)) {
874 fprintf(stderr, "could not create directory 'set'\n");
878 razor_prepare_import(&ctx);
880 for (i = 2; i < argc; i++) {
881 if (razor_import(&ctx, argv[i]) < 0) {
882 fprintf(stderr, "failed to import %s\n",
888 set = razor_finish_import(&ctx);
890 printf("bucket allocation: %d\n", set->buckets.alloc);
891 printf("pool size: %d\n", set->string_pool.size);
892 printf("pool allocation: %d\n", set->string_pool.alloc);
893 printf("packages: %d\n",
894 set->packages.size / sizeof(struct razor_package));
895 printf("requires: %d\n",
896 set->requires.size / sizeof(struct razor_property));
897 printf("provides: %d\n",
898 set->provides.size / sizeof(struct razor_property));
900 razor_set_write(set, repo_filename);
902 razor_set_destroy(set);
903 } else if (strcmp(argv[1], "lookup") == 0) {
904 set = razor_set_open(repo_filename);
905 printf("%s is %lu\n", argv[2],
906 razor_set_lookup(set, argv[2]));
907 razor_set_destroy(set);
908 } else if (strcmp(argv[1], "list") == 0) {
909 set = razor_set_open(repo_filename);
911 razor_set_destroy(set);
912 } else if (strcmp(argv[1], "list-requires") == 0) {
913 set = razor_set_open(repo_filename);
914 razor_set_list_requires(set, argv[2]);
915 razor_set_destroy(set);
916 } else if (strcmp(argv[1], "list-provides") == 0) {
917 set = razor_set_open(repo_filename);
918 razor_set_list_provides(set, argv[2]);
919 razor_set_destroy(set);
920 } else if (strcmp(argv[1], "info") == 0) {
921 set = razor_set_open(repo_filename);
923 razor_set_destroy(set);